computervision/FuseCap.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4",
      "authorship_tag": "ABX9TyPpFW3KjJEzEYsrXI2ohg9M",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/softmurata/colab_notebooks/blob/main/computervision/FuseCap.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Installation"
      ],
      "metadata": {
        "id": "NwEAS5rNs-r2"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "98lLNBsyrAkp"
      },
      "outputs": [],
      "source": [
        "!pip install transformers accelerate bitsandbytes"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Load model"
      ],
      "metadata": {
        "id": "ownhA30FtDAf"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "from transformers import BlipProcessor, BlipForConditionalGeneration\n",
        "\n",
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "\n",
        "processor = BlipProcessor.from_pretrained(\"noamrot/FuseCap\")\n",
        "model = BlipForConditionalGeneration.from_pretrained(\"noamrot/FuseCap\").to(device)"
      ],
      "metadata": {
        "id": "868tyBqIrNwg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def inference(raw_image):\n",
        "    text = \"a picture of \"\n",
        "    inputs = processor(raw_image, text, return_tensors=\"pt\").to(device)\n",
        "    out = model.generate(**inputs)\n",
        "    caption = processor.decode(out[0], skip_special_tokens=True)\n",
        "    return caption"
      ],
      "metadata": {
        "id": "edxUk3JPrsrp"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!wget \"https://www.bhg.com/thmb/Ke48CUYRrXfAc_CJP4eXuZsrWkw=/750x0/filters:no_upscale():max_bytes(150000):strip_icc():format(webp)/living-room-gallery-shelves-l-shaped-couch-ELeyNpyyqpZ8hosOG3EG1X-b5a39646574544e8a75f2961332cd89a.jpg\" -O /content/room.jpg"
      ],
      "metadata": {
        "id": "BQAvNIkcr0bj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from PIL import Image\n",
        "raw_image = Image.open(\"/content/room.jpg\")\n",
        "caption = inference(raw_image)\n",
        "print(caption)"
      ],
      "metadata": {
        "id": "k6IDXLJjrtn1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "display(Image.open(\"/content/room.jpg\"))"
      ],
      "metadata": {
        "id": "sg00qmPrtYbI"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}