Stable_Diffusion_tokenizer_and_embedding_SDA.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/sagiodev/stablediffusion_webui/blob/master/Stable_Diffusion_tokenizer_and_embedding_SDA.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YPHN7PJgKOzb"
      },
      "source": [
        "# Stable Diffusion v1 tokenizer and embedding\n",
        "\n",
        "This notebook examines tokens and embeddings used in Stable Diffusion v1.\n",
        "\n",
        "Tutorials, prompts and resources at https://stable-diffusion-art.com\n",
        "\n",
        "Modified from [Interacting with CLIP notebook](https://colab.research.google.com/github/openai/clip/blob/master/notebooks/Interacting_with_CLIP.ipynb)."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "53N4k0pj_9qL"
      },
      "source": [
        "# Setup\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "0BpdJkdBssk9",
        "outputId": "eef4c06e-844e-47ae-dde8-a51ee9d22678"
      },
      "source": [
        "! pip install ftfy regex tqdm\n",
        "! pip install git+https://github.com/openai/CLIP.git\n",
        "\n",
        "import numpy as np\n",
        "import torch\n",
        "from pkg_resources import packaging\n",
        "print(\"Torch version:\", torch.__version__)\n",
        "\n",
        "import clip\n",
        "print('Available models:')\n",
        "print(clip.available_models())\n",
        "\n",
        "model, preprocess = clip.load(\"ViT-L/14\") # used by stable diffusion v1\n",
        "model.cuda().eval()\n",
        "input_resolution = model.visual.input_resolution\n",
        "context_length = model.context_length\n",
        "vocab_size = model.vocab_size\n",
        "\n",
        "print(\"Model parameters:\", f\"{np.sum([int(np.prod(p.shape)) for p in model.parameters()]):,}\")\n",
        "print(\"Input resolution:\", input_resolution)\n",
        "print(\"Context length:\", context_length)\n",
        "print(\"Vocab size:\", vocab_size)\n"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting ftfy\n",
            "  Downloading ftfy-6.1.3-py3-none-any.whl (53 kB)\n",
            "\u001b[?25l     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/53.4 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K     \u001b[91m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━\u001b[0m \u001b[32m51.2/53.4 kB\u001b[0m \u001b[31m1.5 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m53.4/53.4 kB\u001b[0m \u001b[31m1.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: regex in /usr/local/lib/python3.10/dist-packages (2023.6.3)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (4.66.1)\n",
            "Collecting wcwidth<0.3.0,>=0.2.12 (from ftfy)\n",
            "  Downloading wcwidth-0.2.12-py2.py3-none-any.whl (34 kB)\n",
            "Installing collected packages: wcwidth, ftfy\n",
            "  Attempting uninstall: wcwidth\n",
            "    Found existing installation: wcwidth 0.2.10\n",
            "    Uninstalling wcwidth-0.2.10:\n",
            "      Successfully uninstalled wcwidth-0.2.10\n",
            "Successfully installed ftfy-6.1.3 wcwidth-0.2.12\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "wcwidth"
                ]
              }
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting git+https://github.com/openai/CLIP.git\n",
            "  Cloning https://github.com/openai/CLIP.git to /tmp/pip-req-build-v5rzg3fj\n",
            "  Running command git clone --filter=blob:none --quiet https://github.com/openai/CLIP.git /tmp/pip-req-build-v5rzg3fj\n",
            "  Resolved https://github.com/openai/CLIP.git to commit a1d071733d7111c9c014f024669f959182114e33\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: ftfy in /usr/local/lib/python3.10/dist-packages (from clip==1.0) (6.1.3)\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.10/dist-packages (from clip==1.0) (2023.6.3)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from clip==1.0) (4.66.1)\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.10/dist-packages (from clip==1.0) (2.1.0+cu118)\n",
            "Requirement already satisfied: torchvision in /usr/local/lib/python3.10/dist-packages (from clip==1.0) (0.16.0+cu118)\n",
            "Requirement already satisfied: wcwidth<0.3.0,>=0.2.12 in /usr/local/lib/python3.10/dist-packages (from ftfy->clip==1.0) (0.2.12)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch->clip==1.0) (3.13.1)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from torch->clip==1.0) (4.5.0)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch->clip==1.0) (1.12)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch->clip==1.0) (3.2.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch->clip==1.0) (3.1.2)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch->clip==1.0) (2023.6.0)\n",
            "Requirement already satisfied: triton==2.1.0 in /usr/local/lib/python3.10/dist-packages (from torch->clip==1.0) (2.1.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from torchvision->clip==1.0) (1.23.5)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from torchvision->clip==1.0) (2.31.0)\n",
            "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /usr/local/lib/python3.10/dist-packages (from torchvision->clip==1.0) (9.4.0)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch->clip==1.0) (2.1.3)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->torchvision->clip==1.0) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->torchvision->clip==1.0) (3.4)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->torchvision->clip==1.0) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->torchvision->clip==1.0) (2023.7.22)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch->clip==1.0) (1.3.0)\n",
            "Building wheels for collected packages: clip\n",
            "  Building wheel for clip (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for clip: filename=clip-1.0-py3-none-any.whl size=1369500 sha256=154c892adef7967ec9cdece9f8650d9e8f40286102e9a87e138fba37457581df\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-p9jy548o/wheels/da/2b/4c/d6691fa9597aac8bb85d2ac13b112deb897d5b50f5ad9a37e4\n",
            "Successfully built clip\n",
            "Installing collected packages: clip\n",
            "Successfully installed clip-1.0\n",
            "Torch version: 2.1.0+cu118\n",
            "Available models:\n",
            "['RN50', 'RN101', 'RN50x4', 'RN50x16', 'RN50x64', 'ViT-B/32', 'ViT-B/16', 'ViT-L/14', 'ViT-L/14@336px']\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|███████████████████████████████████████| 890M/890M [00:12<00:00, 76.7MiB/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model parameters: 427,616,513\n",
            "Input resolution: 224\n",
            "Context length: 77\n",
            "Vocab size: 49408\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xwSB5jZki3Cj"
      },
      "source": [
        "# Token and embedding\n",
        "Modify prompt to see tokens and embeddings"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qGom156-i2kL",
        "outputId": "f2fc4cc1-fe3b-4515-9220-55e957f5bfe2"
      },
      "source": [
        "# modify prompt to check tokens\n",
        "prompt = \"Photo of a cat\"\n",
        "\n",
        "tokens = clip.tokenize(prompt)\n",
        "with torch.no_grad():\n",
        "    embeddings = model.token_embedding(tokens.cuda()).float()\n",
        "print(\"text tokens:\")\n",
        "print(tokens)\n",
        "print(\"text tokens size:\", tokens.shape)\n",
        "print(\"Embeddings size:\", embeddings.shape )"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "text tokens:\n",
            "tensor([[49406,  1125,   539,   320,  2368, 49407,     0,     0,     0,     0,\n",
            "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "             0,     0,     0,     0,     0,     0,     0]], dtype=torch.int32)\n",
            "text tokens size: torch.Size([1, 77])\n",
            "Embeddings size: torch.Size([1, 77, 768])\n"
          ]
        }
      ]
    }
  ]
}