diff --git a/docs/extras/integrations/tools/dalle_image_generator.ipynb b/docs/extras/integrations/tools/dalle_image_generator.ipynb index d62d23d551a7f1..ba01e52df33f93 100644 --- a/docs/extras/integrations/tools/dalle_image_generator.ipynb +++ b/docs/extras/integrations/tools/dalle_image_generator.ipynb @@ -1,181 +1,194 @@ { - "cells": [ - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Dall-E Image Generator\n", - "\n", - "This notebook shows how you can generate images from a prompt synthesized using an OpenAI LLM. The images are generated using Dall-E, which uses the same OpenAI API key as the LLM." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Needed if you would like to display images in the notebook\n", - "!pip install opencv-python scikit-image" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "id": "q-k8wmp0zquh" - }, - "outputs": [], - "source": [ - "from langchain.llms import OpenAI\n", - "import os\n", - "os.environ[\"OPENAI_API_KEY\"] = \"\"" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Run as a chain" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "from langchain.utilities.dalle_image_generator import DallEAPIWrapper\n", - "from langchain.prompts import PromptTemplate\n", - "from langchain.chains import LLMChain\n", - "\n", - "llm = OpenAI(temperature=0.9)\n", - "prompt = PromptTemplate(\n", - " input_variables=[\"image_desc\"],\n", - " template=\"Generate a detailed prompt to generate an image based on the following description: {image_desc}\",\n", - ")\n", - "chain = LLMChain(llm=llm, prompt=prompt)" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "https://oaidalleapiprodscus.blob.core.windows.net/private/org-rocrupyvzgcl4yf25rqq6d1v/user-WsxrbKyP2c8rfhCKWDyMfe8N/img-mg1OWiziXxQN1aR2XRsLNndg.png?st=2023-01-31T07%3A34%3A15Z&se=2023-01-31T09%3A34%3A15Z&sp=r&sv=2021-08-06&sr=b&rscd=inline&rsct=image/png&skoid=6aaadede-4fb3-4698-a8f6-684d7786b067&sktid=a48cca56-e6da-484e-a814-9c849652bcb3&skt=2023-01-30T22%3A19%3A44Z&ske=2023-01-31T22%3A19%3A44Z&sks=b&skv=2021-08-06&sig=XDPee5aEng%2BcbXq2mqhh39uHGZTBmJgGAerSd0g%2BMEs%3D\n" - ] - } - ], - "source": [ - "image_url = DallEAPIWrapper().run(chain.run(\"halloween night at a haunted museum\"))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# You can click on the link above to display the image \n", - "# Or you can try the options below to display the image inline in this notebook\n", - "\n", - "try:\n", - " import google.colab\n", - " IN_COLAB = True\n", - "except:\n", - " IN_COLAB = False\n", - "\n", - "if IN_COLAB:\n", - " from google.colab.patches import cv2_imshow # for image display\n", - " from skimage import io\n", - "\n", - " image = io.imread(image_url) \n", - " cv2_imshow(image)\n", - "else:\n", - " import cv2\n", - " from skimage import io\n", - "\n", - " image = io.imread(image_url) \n", - " cv2.imshow('image', image)\n", - " cv2.waitKey(0) #wait for a keyboard input\n", - " cv2.destroyAllWindows()\n" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Run as a tool with an agent" - ] - }, + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Dall-E Image Generator\n", + "\n", + "This notebook shows how you can generate images from a prompt synthesized using an OpenAI LLM. The images are generated using Dall-E, which uses the same OpenAI API key as the LLM." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Needed if you would like to display images in the notebook\n", + "!pip install opencv-python scikit-image" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "id": "q-k8wmp0zquh" + }, + "outputs": [], + "source": [ + "from langchain.llms import OpenAI\n", + "import os\n", + "os.environ[\"OPENAI_API_KEY\"] = \"\"" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Run as a chain" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "from langchain.utilities.dalle_image_generator import DallEAPIWrapper\n", + "from langchain.prompts import PromptTemplate\n", + "from langchain.chains import LLMChain\n", + "from langchain.llms import OpenAI\n", + "\n", + "llm = OpenAI(temperature=0.9)\n", + "prompt = PromptTemplate(\n", + " input_variables=[\"image_desc\"],\n", + " template=\"Generate a detailed prompt to generate an image based on the following description: {image_desc}\",\n", + ")\n", + "chain = LLMChain(llm=llm, prompt=prompt)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "image_url = DallEAPIWrapper().run(chain.run(\"halloween night at a haunted museum\"))" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "\n", - "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n", - "\u001b[32;1m\u001b[1;3m What is the best way to turn this description into an image?\n", - "Action: Dall-E Image Generator\n", - "Action Input: A spooky Halloween night at a haunted museum\u001b[0mhttps://oaidalleapiprodscus.blob.core.windows.net/private/org-rocrupyvzgcl4yf25rqq6d1v/user-WsxrbKyP2c8rfhCKWDyMfe8N/img-ogKfqxxOS5KWVSj4gYySR6FY.png?st=2023-01-31T07%3A38%3A25Z&se=2023-01-31T09%3A38%3A25Z&sp=r&sv=2021-08-06&sr=b&rscd=inline&rsct=image/png&skoid=6aaadede-4fb3-4698-a8f6-684d7786b067&sktid=a48cca56-e6da-484e-a814-9c849652bcb3&skt=2023-01-30T22%3A19%3A36Z&ske=2023-01-31T22%3A19%3A36Z&sks=b&skv=2021-08-06&sig=XsomxxBfu2CP78SzR9lrWUlbask4wBNnaMsHamy4VvU%3D\n", - "\n", - "Observation: \u001b[36;1m\u001b[1;3mhttps://oaidalleapiprodscus.blob.core.windows.net/private/org-rocrupyvzgcl4yf25rqq6d1v/user-WsxrbKyP2c8rfhCKWDyMfe8N/img-ogKfqxxOS5KWVSj4gYySR6FY.png?st=2023-01-31T07%3A38%3A25Z&se=2023-01-31T09%3A38%3A25Z&sp=r&sv=2021-08-06&sr=b&rscd=inline&rsct=image/png&skoid=6aaadede-4fb3-4698-a8f6-684d7786b067&sktid=a48cca56-e6da-484e-a814-9c849652bcb3&skt=2023-01-30T22%3A19%3A36Z&ske=2023-01-31T22%3A19%3A36Z&sks=b&skv=2021-08-06&sig=XsomxxBfu2CP78SzR9lrWUlbask4wBNnaMsHamy4VvU%3D\u001b[0m\n", - "Thought:\u001b[32;1m\u001b[1;3m With the image generated, I can now make my final answer.\n", - "Final Answer: An image of a Halloween night at a haunted museum can be seen here: https://oaidalleapiprodscus.blob.core.windows.net/private/org-rocrupyvzgcl4yf25rqq6d1v/user-WsxrbKyP2c8rfhCKWDyMfe8N/img-ogKfqxxOS5KWVSj4gYySR6FY.png?st=2023-01-31T07%3A38%3A25Z&se=2023-01-31T09%3A38%3A25Z&sp=r&sv=2021-08-06&sr=b&rscd=inline&rsct=image/png&skoid=6aaadede-4fb3-4698-a8f6-684d7786b067&sktid=a48cca56-e6da-484e-a814-9c849652bcb3&skt=2023-01-30T22\u001b[0m\n", - "\n", - "\u001b[1m> Finished chain.\u001b[0m\n" - ] - } - ], - "source": [ - "from langchain.agents import load_tools\n", - "from langchain.agents import initialize_agent\n", - "\n", - "tools = load_tools(['dalle-image-generator'])\n", - "agent = initialize_agent(tools, llm, agent=\"zero-shot-react-description\", verbose=True)\n", - "output = agent.run(\"Create an image of a halloween night at a haunted museum\")" + "data": { + "text/plain": [ + "'https://oaidalleapiprodscus.blob.core.windows.net/private/org-i0zjYONU3PemzJ222esBaAzZ/user-f6uEIOFxoiUZivy567cDSWni/img-i7Z2ZxvJ4IbbdAiO6OXJgS3v.png?st=2023-08-11T14%3A03%3A14Z&se=2023-08-11T16%3A03%3A14Z&sp=r&sv=2021-08-06&sr=b&rscd=inline&rsct=image/png&skoid=6aaadede-4fb3-4698-a8f6-684d7786b067&sktid=a48cca56-e6da-484e-a814-9c849652bcb3&skt=2023-08-10T20%3A58%3A32Z&ske=2023-08-11T20%3A58%3A32Z&sks=b&skv=2021-08-06&sig=/sECe7C0EAq37ssgBm7g7JkVIM/Q1W3xOstd0Go6slA%3D'" ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "langchain", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.16" - }, - "vscode": { - "interpreter": { - "hash": "3570c8892273ffbeee7ead61dc7c022b73551d9f55fb2584ac0e8e8920b18a89" - } + ], + "source": [ + "image_url" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# You can click on the link above to display the image \n", + "# Or you can try the options below to display the image inline in this notebook\n", + "\n", + "try:\n", + " import google.colab\n", + " IN_COLAB = True\n", + "except:\n", + " IN_COLAB = False\n", + "\n", + "if IN_COLAB:\n", + " from google.colab.patches import cv2_imshow # for image display\n", + " from skimage import io\n", + "\n", + " image = io.imread(image_url) \n", + " cv2_imshow(image)\n", + "else:\n", + " import cv2\n", + " from skimage import io\n", + "\n", + " image = io.imread(image_url) \n", + " cv2.imshow('image', image)\n", + " cv2.waitKey(0) #wait for a keyboard input\n", + " cv2.destroyAllWindows()\n" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Run as a tool with an agent" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n", + "\u001b[32;1m\u001b[1;3m What is the best way to turn this description into an image?\n", + "Action: Dall-E Image Generator\n", + "Action Input: A spooky Halloween night at a haunted museum\u001b[0mhttps://oaidalleapiprodscus.blob.core.windows.net/private/org-rocrupyvzgcl4yf25rqq6d1v/user-WsxrbKyP2c8rfhCKWDyMfe8N/img-ogKfqxxOS5KWVSj4gYySR6FY.png?st=2023-01-31T07%3A38%3A25Z&se=2023-01-31T09%3A38%3A25Z&sp=r&sv=2021-08-06&sr=b&rscd=inline&rsct=image/png&skoid=6aaadede-4fb3-4698-a8f6-684d7786b067&sktid=a48cca56-e6da-484e-a814-9c849652bcb3&skt=2023-01-30T22%3A19%3A36Z&ske=2023-01-31T22%3A19%3A36Z&sks=b&skv=2021-08-06&sig=XsomxxBfu2CP78SzR9lrWUlbask4wBNnaMsHamy4VvU%3D\n", + "\n", + "Observation: \u001b[36;1m\u001b[1;3mhttps://oaidalleapiprodscus.blob.core.windows.net/private/org-rocrupyvzgcl4yf25rqq6d1v/user-WsxrbKyP2c8rfhCKWDyMfe8N/img-ogKfqxxOS5KWVSj4gYySR6FY.png?st=2023-01-31T07%3A38%3A25Z&se=2023-01-31T09%3A38%3A25Z&sp=r&sv=2021-08-06&sr=b&rscd=inline&rsct=image/png&skoid=6aaadede-4fb3-4698-a8f6-684d7786b067&sktid=a48cca56-e6da-484e-a814-9c849652bcb3&skt=2023-01-30T22%3A19%3A36Z&ske=2023-01-31T22%3A19%3A36Z&sks=b&skv=2021-08-06&sig=XsomxxBfu2CP78SzR9lrWUlbask4wBNnaMsHamy4VvU%3D\u001b[0m\n", + "Thought:\u001b[32;1m\u001b[1;3m With the image generated, I can now make my final answer.\n", + "Final Answer: An image of a Halloween night at a haunted museum can be seen here: https://oaidalleapiprodscus.blob.core.windows.net/private/org-rocrupyvzgcl4yf25rqq6d1v/user-WsxrbKyP2c8rfhCKWDyMfe8N/img-ogKfqxxOS5KWVSj4gYySR6FY.png?st=2023-01-31T07%3A38%3A25Z&se=2023-01-31T09%3A38%3A25Z&sp=r&sv=2021-08-06&sr=b&rscd=inline&rsct=image/png&skoid=6aaadede-4fb3-4698-a8f6-684d7786b067&sktid=a48cca56-e6da-484e-a814-9c849652bcb3&skt=2023-01-30T22\u001b[0m\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n" + ] } + ], + "source": [ + "from langchain.agents import load_tools\n", + "from langchain.agents import initialize_agent\n", + "\n", + "tools = load_tools(['dalle-image-generator'])\n", + "agent = initialize_agent(tools, llm, agent=\"zero-shot-react-description\", verbose=True)\n", + "output = agent.run(\"Create an image of a halloween night at a haunted museum\")" + ] + } + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "poetry-venv", + "language": "python", + "name": "poetry-venv" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.1" }, - "nbformat": 4, - "nbformat_minor": 0 + "vscode": { + "interpreter": { + "hash": "3570c8892273ffbeee7ead61dc7c022b73551d9f55fb2584ac0e8e8920b18a89" + } + } + }, + "nbformat": 4, + "nbformat_minor": 4 }