DeepSeek_VL_Gradio_Multimodal.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "machine_shape": "hm",
      "gpuType": "T4",
      "authorship_tag": "ABX9TyMzH9lflqv3ES+7PA10A68Y",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/yvrjsharma/HugginFace_Gradio/blob/main/DeepSeek_VL_Gradio_Multimodal.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# DeepSeek-VL with Gradio Multimodal Chatbots"
      ],
      "metadata": {
        "id": "BUbCHXLb68Q-"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Install gradio"
      ],
      "metadata": {
        "id": "51T5zswe7C4A"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JxeH3BnFv3w_",
        "outputId": "5724edc4-9fb2-4bf2-a8c8-e73e247abbd4"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m17.0/17.0 MB\u001b[0m \u001b[31m81.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m92.1/92.1 kB\u001b[0m \u001b[31m12.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m310.7/310.7 kB\u001b[0m \u001b[31m32.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.6/75.6 kB\u001b[0m \u001b[31m7.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m138.5/138.5 kB\u001b[0m \u001b[31m17.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.9/7.9 MB\u001b[0m \u001b[31m94.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m60.6/60.6 kB\u001b[0m \u001b[31m7.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m129.9/129.9 kB\u001b[0m \u001b[31m14.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.8/77.8 kB\u001b[0m \u001b[31m10.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m7.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m71.5/71.5 kB\u001b[0m \u001b[31m9.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Building wheel for ffmpy (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
          ]
        }
      ],
      "source": [
        "!pip install gradio -q\n",
        "import gradio as gr"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Clone DeepSeek-VL lib"
      ],
      "metadata": {
        "id": "hLflWXMi7GF9"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!git clone https://github.com/deepseek-ai/DeepSeek-VL\n",
        "%cd DeepSeek-VL\n",
        "\n",
        "!pip install -e ."
      ],
      "metadata": {
        "id": "QvFPT0jCwID9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#no\n",
        "!pip install typing-extensions"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "r09WOj4JxWAa",
        "outputId": "8cdadafb-57ad-40e9-df06-a6d1e5fe8c6c"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (4.10.0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Import libraries"
      ],
      "metadata": {
        "id": "9U8eTz1x7WHF"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import collections.abc\n",
        "import sys\n",
        "import torch\n",
        "from transformers import AutoModelForCausalLM\n",
        "\n",
        "# Temporarily fix the ImportError for collections.Mapping\n",
        "sys.modules['collections.Mapping'] = collections.abc.Mapping\n",
        "sys.modules['collections.MutableMapping'] = collections.abc.MutableMapping\n",
        "sys.modules['collections.Sequence'] = collections.abc.Sequence\n",
        "\n",
        "# Now try importing the deepseek_vl package\n",
        "from deepseek_vl.models import VLChatProcessor, MultiModalityCausalLM\n",
        "from deepseek_vl.utils.io import load_pil_images\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dJ7btZpwyZj4",
        "outputId": "df200778-b7db-4f92-fb0b-01b73e72ffb4"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/torchvision/io/image.py:13: UserWarning: Failed to load image Python extension: '/usr/local/lib/python3.10/dist-packages/torchvision/image.so: undefined symbol: _ZN3c1017RegisterOperatorsD1Ev'If you don't plan on using image functionality from `torchvision.io`, you can ignore this warning. Otherwise, there might be something wrong with your environment. Did you have `libjpeg` or `libpng` installed before building `torchvision` from source?\n",
            "  warn(\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Download and initialize 1.3b VLM chat model"
      ],
      "metadata": {
        "id": "1Y4G2n3J73VG"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# specify the path to the model\n",
        "model_path = \"deepseek-ai/deepseek-vl-1.3b-chat\"\n",
        "vl_chat_processor: VLChatProcessor = VLChatProcessor.from_pretrained(model_path)\n",
        "tokenizer = vl_chat_processor.tokenizer\n",
        "\n",
        "vl_gpt: MultiModalityCausalLM = AutoModelForCausalLM.from_pretrained(model_path, trust_remote_code=True)\n",
        "vl_gpt = vl_gpt.to(torch.bfloat16).cuda().eval()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qr4A8QE6wBog",
        "outputId": "cb5eb520-ac22-46d5-c3fc-d4e68aba9ba9"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:88: UserWarning: \n",
            "The secret `HF_TOKEN` does not exist in your Colab secrets.\n",
            "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n",
            "You will be able to reuse this secret in all of your notebooks.\n",
            "Please note that authentication is recommended but still optional to access public models or datasets.\n",
            "  warnings.warn(\n",
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Chat Prompt"
      ],
      "metadata": {
        "id": "mHGTZYhH8tFQ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "conversation = [\n",
        "    {\n",
        "        \"role\": \"User\",\n",
        "        \"content\": \"<image_placeholder>Describe each stage of this image.\",\n",
        "        \"images\": [\"/content/DeepSeek-VL/images/training_pipelines.png\"]\n",
        "    },\n",
        "    {\n",
        "        \"role\": \"Assistant\",\n",
        "        \"content\": \"\"\n",
        "    }\n",
        "]"
      ],
      "metadata": {
        "id": "ER9by_xpwOJl"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Define graio predict function for chat inference"
      ],
      "metadata": {
        "id": "cVzuUw8O89S7"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "def deepseekvl(messages):\n",
        "\n",
        "  pil_images = load_pil_images(messages)\n",
        "  prepare_inputs = vl_chat_processor(\n",
        "      conversations=messages,\n",
        "      images=pil_images,\n",
        "      force_batchify=True\n",
        "  ).to(vl_gpt.device)\n",
        "  # run image encoder to get the image embeddings\n",
        "  inputs_embeds = vl_gpt.prepare_inputs_embeds(**prepare_inputs)\n",
        "  # run the model to get the response\n",
        "  outputs = vl_gpt.language_model.generate(\n",
        "      inputs_embeds=inputs_embeds,\n",
        "      attention_mask=prepare_inputs.attention_mask,\n",
        "      pad_token_id=tokenizer.eos_token_id,\n",
        "      bos_token_id=tokenizer.bos_token_id,\n",
        "      eos_token_id=tokenizer.eos_token_id,\n",
        "      max_new_tokens=512,\n",
        "      do_sample=False,\n",
        "      use_cache=True)\n",
        "\n",
        "  answer = tokenizer.decode(outputs[0].cpu().tolist(), skip_special_tokens=True)\n",
        "  return answer\n",
        "\n",
        "\n",
        "def bot(msg, history):\n",
        "\n",
        "    messages = []\n",
        "    history = history + [[msg, None]]\n",
        "    history_len = len(history)\n",
        "    image_flag=False\n",
        "    for idx, (user_message, assistant_message) in enumerate(history):\n",
        "\n",
        "      if idx == (history_len-1):\n",
        "        break\n",
        "      if image_flag:\n",
        "        image_flag=False\n",
        "        continue\n",
        "      if isinstance(user_message, tuple):\n",
        "        if idx == (history_len-2):\n",
        "          messages.append({\"role\": \"user\",\n",
        "                          \"content\": f\"<image_placeholder>{msg}\",\n",
        "                          \"images\": [user_message[0]],})\n",
        "          messages.append({\"role\": \"Assistant\", \"content\": \"\"})\n",
        "          image_flag=True\n",
        "          response = deepseekvl(messages)\n",
        "          history[-1][1] = response\n",
        "          return history, ''\n",
        "        else:\n",
        "          messages.append({\"role\": \"user\",\n",
        "                           \"content\": f\"<image_placeholder>{history[idx+1][0]}\",\n",
        "                           \"images\": [user_message[0]],})\n",
        "          image_flag=True\n",
        "      else:\n",
        "        messages.append({\"role\": \"user\", \"content\": user_message})\n",
        "        messages.append({\"role\": \"assistant\", \"content\": assistant_message})\n",
        "\n",
        "    messages.append({\"role\": \"user\", \"content\": msg})\n",
        "    messages.append({\"role\": \"Assistant\", \"content\": \"\"})\n",
        "    response = deepseekvl(messages)\n",
        "    history[-1][1] = response\n",
        "    return history, ''\n"
      ],
      "metadata": {
        "id": "Q6P-J92N_CMQ"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Run this only if you get the below error while running inference in Gradio chatbot:\n",
        "\n",
        "> RuntimeError: cutlassF: no kernel found to launch\n",
        "\n",
        "Restart the session after this"
      ],
      "metadata": {
        "id": "fE-bNGma-QnY"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install torch --upgrade"
      ],
      "metadata": {
        "id": "PwSC98Rx46cZ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## **Gradio layout**"
      ],
      "metadata": {
        "id": "N37UYyXj-ann"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import gradio as gr\n",
        "import os\n",
        "import time\n",
        "\n",
        "# Chatbot demo with multimodal input\n",
        "\n",
        "def print_like_dislike(x: gr.LikeData):\n",
        "    # for now just print the messages liked or disliked by a user\n",
        "    # you can implement a more complex workflow here as well\n",
        "    print(x.index, x.value, x.liked)\n",
        "\n",
        "# upload file to chatbot\n",
        "def add_file(history, file):\n",
        "    history = history + [[(file.name,), None]]\n",
        "    return history\n",
        "\n",
        "\n",
        "with gr.Blocks(fill_height=True) as demo:\n",
        "    gr.HTML(\"\"\"<h1><center> DeepSeek-VL (1.3B Parameters) </center></h1>\"\"\")\n",
        "    chatbot = gr.Chatbot(\n",
        "        [],\n",
        "        elem_id=\"chatbot\",\n",
        "        bubble_full_width=False,\n",
        "        #avatar_images=(None, (\"/content/deepseeklogo.png\")),\n",
        "    )\n",
        "\n",
        "    with gr.Row():\n",
        "        txt = gr.Textbox(\n",
        "            scale=4,\n",
        "            show_label=False,\n",
        "            placeholder=\"Enter text and press enter, or upload an image\",\n",
        "            container=False,\n",
        "        )\n",
        "        btn = gr.UploadButton(\"📁\", file_types=[\"image\", \"video\", \"audio\"])\n",
        "\n",
        "    txt_msg = txt.submit(bot, [txt, chatbot], [chatbot,txt] )\n",
        "\n",
        "    txt_msg.then(lambda: gr.Textbox(interactive=True), None, [txt], queue=False)\n",
        "    file_msg = btn.upload(add_file, [chatbot, btn], [chatbot], queue=False)\n",
        "\n",
        "    chatbot.like(print_like_dislike, None, None)\n",
        "\n",
        "\n",
        "demo.queue()\n",
        "demo.launch(debug=True,)\n",
        "\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 690
        },
        "id": "aJNFRshN7vW7",
        "outputId": "d66eb7ed-0149-4928-e4e8-d94516a9d977"
      },
      "execution_count": 11,
      "outputs": [
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Setting queue=True in a Colab notebook requires sharing enabled. Setting `share=True` (you can turn this off by setting `share=False` in `launch()` explicitly).\n",
            "\n",
            "Colab notebook detected. This cell will run indefinitely so that you can see errors and logs. To turn off, set debug=False in launch().\n",
            "Running on public URL: https://c07cb48b1f0ba7903b.gradio.live\n",
            "\n",
            "This share link expires in 72 hours. For free permanent hosting and GPU upgrades, run `gradio deploy` from Terminal to deploy to Spaces (https://huggingface.co/spaces)\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<div><iframe src=\"https://c07cb48b1f0ba7903b.gradio.live\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Keyboard interruption in main thread... closing server.\n",
            "Killing tunnel 127.0.0.1:7860 <> https://c07cb48b1f0ba7903b.gradio.live\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": []
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "9"
      ],
      "metadata": {
        "id": "EJJoECBC603K"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}