06- Get CLIP_results_sep.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "33d3c9c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "#from pathlib import Path\n",
    "\n",
    "path_data = \"/data\"\n",
    "path_original_images = 'data/original_images'\n",
    "path_processed_images = \"/\".join([path_data, \"processed_imgs\"])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "57dc5821",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Torch version: 1.10.2+cu113\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns \n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "import os\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "import torch\n",
    "from pkg_resources import packaging\n",
    "from PIL import Image\n",
    "import tqdm\n",
    "from pathlib import Path\n",
    "print(\"Torch version:\", torch.__version__)\n",
    "\n",
    "\n",
    "plt.style.use('ggplot')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "d97af734",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>country.name</th>\n",
       "      <th>country.id</th>\n",
       "      <th>region.id</th>\n",
       "      <th>type</th>\n",
       "      <th>imageRelPath</th>\n",
       "      <th>topics</th>\n",
       "      <th>place</th>\n",
       "      <th>income</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5d4be6cecf0b3a0f3f344586</td>\n",
       "      <td>Burundi</td>\n",
       "      <td>bi</td>\n",
       "      <td>af</td>\n",
       "      <td>image</td>\n",
       "      <td>assets/5d4be6cecf0b3a0f3f344586/5d4be6cecf0b3a...</td>\n",
       "      <td>icons</td>\n",
       "      <td>butoyi</td>\n",
       "      <td>26.994581</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5d4bf31ccf0b3a0f3f359814</td>\n",
       "      <td>Burundi</td>\n",
       "      <td>bi</td>\n",
       "      <td>af</td>\n",
       "      <td>image</td>\n",
       "      <td>assets/5d4bf31ccf0b3a0f3f359814/5d4bf31ccf0b3a...</td>\n",
       "      <td>Family snapshots</td>\n",
       "      <td>butoyi</td>\n",
       "      <td>26.994581</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5d4bf31ccf0b3a0f3f35982a</td>\n",
       "      <td>Burundi</td>\n",
       "      <td>bi</td>\n",
       "      <td>af</td>\n",
       "      <td>image</td>\n",
       "      <td>assets/5d4bf31ccf0b3a0f3f35982a/5d4bf31ccf0b3a...</td>\n",
       "      <td>Cutlery</td>\n",
       "      <td>butoyi</td>\n",
       "      <td>26.994581</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5d4bf31ccf0b3a0f3f35982e</td>\n",
       "      <td>Burundi</td>\n",
       "      <td>bi</td>\n",
       "      <td>af</td>\n",
       "      <td>image</td>\n",
       "      <td>assets/5d4bf31ccf0b3a0f3f35982e/5d4bf31ccf0b3a...</td>\n",
       "      <td>Family</td>\n",
       "      <td>butoyi</td>\n",
       "      <td>26.994581</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5d4bf31ccf0b3a0f3f359830</td>\n",
       "      <td>Burundi</td>\n",
       "      <td>bi</td>\n",
       "      <td>af</td>\n",
       "      <td>image</td>\n",
       "      <td>assets/5d4bf31ccf0b3a0f3f359830/5d4bf31ccf0b3a...</td>\n",
       "      <td>Place where eating dinner</td>\n",
       "      <td>butoyi</td>\n",
       "      <td>26.994581</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         id country.name country.id region.id   type  \\\n",
       "0  5d4be6cecf0b3a0f3f344586      Burundi         bi        af  image   \n",
       "1  5d4bf31ccf0b3a0f3f359814      Burundi         bi        af  image   \n",
       "2  5d4bf31ccf0b3a0f3f35982a      Burundi         bi        af  image   \n",
       "3  5d4bf31ccf0b3a0f3f35982e      Burundi         bi        af  image   \n",
       "4  5d4bf31ccf0b3a0f3f359830      Burundi         bi        af  image   \n",
       "\n",
       "                                        imageRelPath  \\\n",
       "0  assets/5d4be6cecf0b3a0f3f344586/5d4be6cecf0b3a...   \n",
       "1  assets/5d4bf31ccf0b3a0f3f359814/5d4bf31ccf0b3a...   \n",
       "2  assets/5d4bf31ccf0b3a0f3f35982a/5d4bf31ccf0b3a...   \n",
       "3  assets/5d4bf31ccf0b3a0f3f35982e/5d4bf31ccf0b3a...   \n",
       "4  assets/5d4bf31ccf0b3a0f3f359830/5d4bf31ccf0b3a...   \n",
       "\n",
       "                      topics   place     income  \n",
       "0                      icons  butoyi  26.994581  \n",
       "1           Family snapshots  butoyi  26.994581  \n",
       "2                    Cutlery  butoyi  26.994581  \n",
       "3                     Family  butoyi  26.994581  \n",
       "4  Place where eating dinner  butoyi  26.994581  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv('data/images_v2.csv')\n",
    "data = data[data.id != '5d4befbfcf0b3a0f3f353c2e'] #remove rows with corrupted image \n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "230550a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "#  list of subjective topics\n",
    "subjective_topics = ['favorite home decorations', 'favourite item in kitchen', \n",
    "                    'favourite sports clubs', 'how the most loved item is used',\n",
    "                    'icons', 'idols', 'latest furniture bought', 'looking over the shoulder',\n",
    "                    'most loved item', 'most loved toy', 'most played songs on the radio',\n",
    "                    'music idol', 'next big thing you are planning to buy',\n",
    "                     'playing with most loved toy',\n",
    "                    'thing i dream about having','things i wish i had',\n",
    "                     'using most loved item', 'youth culture', 'what i wish i could buy']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "6e0bb59b",
   "metadata": {},
   "outputs": [],
   "source": [
    "templates = [\n",
    "    'an image of {}.',\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "47c06c4e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "270\n"
     ]
    }
   ],
   "source": [
    "# map each topic to list of corresponding image ids\n",
    "dict_topic2img = {}\n",
    "dict_id2country = {}\n",
    "for list_topics, image_id in zip(data['topics'], data['id']):\n",
    "    for topic in list_topics.split(\",\"):\n",
    "        topic = topic.lower().strip()\n",
    "        if topic not in subjective_topics: # remove subjectives\n",
    "            if topic not in dict_topic2img:\n",
    "                dict_topic2img[topic] = set()\n",
    "            dict_topic2img[topic].add(image_id)\n",
    "\n",
    "print(len(dict_topic2img))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "4afca63d",
   "metadata": {},
   "outputs": [],
   "source": [
    "list_img_path = []\n",
    "list_topic = []\n",
    "\n",
    "for topic, img_lst in dict_topic2img.items():\n",
    "    for path in img_lst:\n",
    "        list_img_path.append(path)\n",
    "        list_topic.append(topic)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "b099a4df",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "46133"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(list_img_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "3fe369e7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Requirement already satisfied: ftfy in /home/jnwatu/.local/lib/python3.9/site-packages (6.1.1)\n",
      "Requirement already satisfied: regex in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (2021.8.3)\n",
      "Requirement already satisfied: tqdm in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (4.62.3)\n",
      "Requirement already satisfied: wcwidth>=0.2.5 in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from ftfy) (0.2.5)\n",
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Collecting git+https://github.com/openai/CLIP.git\n",
      "  Cloning https://github.com/openai/CLIP.git to /tmp/pip-req-build-h59kz773\n",
      "  Running command git clone -q https://github.com/openai/CLIP.git /tmp/pip-req-build-h59kz773\n",
      "  Resolved https://github.com/openai/CLIP.git to commit a1d071733d7111c9c014f024669f959182114e33\n",
      "Requirement already satisfied: ftfy in /home/jnwatu/.local/lib/python3.9/site-packages (from clip==1.0) (6.1.1)\n",
      "Requirement already satisfied: regex in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from clip==1.0) (2021.8.3)\n",
      "Requirement already satisfied: tqdm in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from clip==1.0) (4.62.3)\n",
      "Requirement already satisfied: torch in /home/jnwatu/.local/lib/python3.9/site-packages (from clip==1.0) (1.10.2+cu113)\n",
      "Requirement already satisfied: torchvision in /home/jnwatu/.local/lib/python3.9/site-packages (from clip==1.0) (0.11.3+cu113)\n",
      "Requirement already satisfied: wcwidth>=0.2.5 in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from ftfy->clip==1.0) (0.2.5)\n",
      "Requirement already satisfied: typing-extensions in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from torch->clip==1.0) (3.10.0.2)\n",
      "Requirement already satisfied: numpy in /home/jnwatu/.local/lib/python3.9/site-packages (from torchvision->clip==1.0) (1.22.4)\n",
      "Requirement already satisfied: pillow!=8.3.0,>=5.3.0 in /sw/pkgs/arc/python3.9-anaconda/2021.11/lib/python3.9/site-packages (from torchvision->clip==1.0) (8.4.0)\n"
     ]
    }
   ],
   "source": [
    "! pip install ftfy regex tqdm\n",
    "! pip install git+https://github.com/openai/CLIP.git"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "82ea3eb4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['RN50',\n",
       " 'RN101',\n",
       " 'RN50x4',\n",
       " 'RN50x16',\n",
       " 'RN50x64',\n",
       " 'ViT-B/32',\n",
       " 'ViT-B/16',\n",
       " 'ViT-L/14',\n",
       " 'ViT-L/14@336px']"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import clip\n",
    "clip.available_models()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "f45af992",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CLIP(\n",
       "  (visual): VisionTransformer(\n",
       "    (conv1): Conv2d(3, 768, kernel_size=(32, 32), stride=(32, 32), bias=False)\n",
       "    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "    (transformer): Transformer(\n",
       "      (resblocks): Sequential(\n",
       "        (0): ResidualAttentionBlock(\n",
       "          (attn): MultiheadAttention(\n",
       "            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)\n",
       "          )\n",
       "          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          (mlp): Sequential(\n",
       "            (c_fc): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (gelu): QuickGELU()\n",
       "            (c_proj): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          )\n",
       "          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "        (1): ResidualAttentionBlock(\n",
       "          (attn): MultiheadAttention(\n",
       "            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)\n",
       "          )\n",
       "          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          (mlp): Sequential(\n",
       "            (c_fc): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (gelu): QuickGELU()\n",
       "            (c_proj): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          )\n",
       "          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "        (2): ResidualAttentionBlock(\n",
       "          (attn): MultiheadAttention(\n",
       "            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)\n",
       "          )\n",
       "          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          (mlp): Sequential(\n",
       "            (c_fc): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (gelu): QuickGELU()\n",
       "            (c_proj): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          )\n",
       "          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "        (3): ResidualAttentionBlock(\n",
       "          (attn): MultiheadAttention(\n",
       "            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)\n",
       "          )\n",
       "          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          (mlp): Sequential(\n",
       "            (c_fc): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (gelu): QuickGELU()\n",
       "            (c_proj): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          )\n",
       "          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "        (4): ResidualAttentionBlock(\n",
       "          (attn): MultiheadAttention(\n",
       "            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)\n",
       "          )\n",
       "          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          (mlp): Sequential(\n",
       "            (c_fc): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (gelu): QuickGELU()\n",
       "            (c_proj): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          )\n",
       "          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "        (5): ResidualAttentionBlock(\n",
       "          (attn): MultiheadAttention(\n",
       "            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)\n",
       "          )\n",
       "          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          (mlp): Sequential(\n",
       "            (c_fc): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (gelu): QuickGELU()\n",
       "            (c_proj): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          )\n",
       "          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "        (6): ResidualAttentionBlock(\n",
       "          (attn): MultiheadAttention(\n",
       "            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)\n",
       "          )\n",
       "          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          (mlp): Sequential(\n",
       "            (c_fc): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (gelu): QuickGELU()\n",
       "            (c_proj): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          )\n",
       "          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "        (7): ResidualAttentionBlock(\n",
       "          (attn): MultiheadAttention(\n",
       "            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)\n",
       "          )\n",
       "          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          (mlp): Sequential(\n",
       "            (c_fc): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (gelu): QuickGELU()\n",
       "            (c_proj): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          )\n",
       "          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "        (8): ResidualAttentionBlock(\n",
       "          (attn): MultiheadAttention(\n",
       "            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)\n",
       "          )\n",
       "          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          (mlp): Sequential(\n",
       "            (c_fc): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (gelu): QuickGELU()\n",
       "            (c_proj): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          )\n",
       "          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "        (9): ResidualAttentionBlock(\n",
       "          (attn): MultiheadAttention(\n",
       "            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)\n",
       "          )\n",
       "          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          (mlp): Sequential(\n",
       "            (c_fc): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (gelu): QuickGELU()\n",
       "            (c_proj): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          )\n",
       "          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "        (10): ResidualAttentionBlock(\n",
       "          (attn): MultiheadAttention(\n",
       "            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)\n",
       "          )\n",
       "          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          (mlp): Sequential(\n",
       "            (c_fc): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (gelu): QuickGELU()\n",
       "            (c_proj): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          )\n",
       "          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "        (11): ResidualAttentionBlock(\n",
       "          (attn): MultiheadAttention(\n",
       "            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)\n",
       "          )\n",
       "          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "          (mlp): Sequential(\n",
       "            (c_fc): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (gelu): QuickGELU()\n",
       "            (c_proj): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          )\n",
       "          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "  )\n",
       "  (transformer): Transformer(\n",
       "    (resblocks): Sequential(\n",
       "      (0): ResidualAttentionBlock(\n",
       "        (attn): MultiheadAttention(\n",
       "          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n",
       "        )\n",
       "        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): Sequential(\n",
       "          (c_fc): Linear(in_features=512, out_features=2048, bias=True)\n",
       "          (gelu): QuickGELU()\n",
       "          (c_proj): Linear(in_features=2048, out_features=512, bias=True)\n",
       "        )\n",
       "        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "      )\n",
       "      (1): ResidualAttentionBlock(\n",
       "        (attn): MultiheadAttention(\n",
       "          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n",
       "        )\n",
       "        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): Sequential(\n",
       "          (c_fc): Linear(in_features=512, out_features=2048, bias=True)\n",
       "          (gelu): QuickGELU()\n",
       "          (c_proj): Linear(in_features=2048, out_features=512, bias=True)\n",
       "        )\n",
       "        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "      )\n",
       "      (2): ResidualAttentionBlock(\n",
       "        (attn): MultiheadAttention(\n",
       "          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n",
       "        )\n",
       "        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): Sequential(\n",
       "          (c_fc): Linear(in_features=512, out_features=2048, bias=True)\n",
       "          (gelu): QuickGELU()\n",
       "          (c_proj): Linear(in_features=2048, out_features=512, bias=True)\n",
       "        )\n",
       "        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "      )\n",
       "      (3): ResidualAttentionBlock(\n",
       "        (attn): MultiheadAttention(\n",
       "          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n",
       "        )\n",
       "        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): Sequential(\n",
       "          (c_fc): Linear(in_features=512, out_features=2048, bias=True)\n",
       "          (gelu): QuickGELU()\n",
       "          (c_proj): Linear(in_features=2048, out_features=512, bias=True)\n",
       "        )\n",
       "        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "      )\n",
       "      (4): ResidualAttentionBlock(\n",
       "        (attn): MultiheadAttention(\n",
       "          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n",
       "        )\n",
       "        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): Sequential(\n",
       "          (c_fc): Linear(in_features=512, out_features=2048, bias=True)\n",
       "          (gelu): QuickGELU()\n",
       "          (c_proj): Linear(in_features=2048, out_features=512, bias=True)\n",
       "        )\n",
       "        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "      )\n",
       "      (5): ResidualAttentionBlock(\n",
       "        (attn): MultiheadAttention(\n",
       "          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n",
       "        )\n",
       "        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): Sequential(\n",
       "          (c_fc): Linear(in_features=512, out_features=2048, bias=True)\n",
       "          (gelu): QuickGELU()\n",
       "          (c_proj): Linear(in_features=2048, out_features=512, bias=True)\n",
       "        )\n",
       "        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "      )\n",
       "      (6): ResidualAttentionBlock(\n",
       "        (attn): MultiheadAttention(\n",
       "          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n",
       "        )\n",
       "        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): Sequential(\n",
       "          (c_fc): Linear(in_features=512, out_features=2048, bias=True)\n",
       "          (gelu): QuickGELU()\n",
       "          (c_proj): Linear(in_features=2048, out_features=512, bias=True)\n",
       "        )\n",
       "        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "      )\n",
       "      (7): ResidualAttentionBlock(\n",
       "        (attn): MultiheadAttention(\n",
       "          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n",
       "        )\n",
       "        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): Sequential(\n",
       "          (c_fc): Linear(in_features=512, out_features=2048, bias=True)\n",
       "          (gelu): QuickGELU()\n",
       "          (c_proj): Linear(in_features=2048, out_features=512, bias=True)\n",
       "        )\n",
       "        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "      )\n",
       "      (8): ResidualAttentionBlock(\n",
       "        (attn): MultiheadAttention(\n",
       "          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n",
       "        )\n",
       "        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): Sequential(\n",
       "          (c_fc): Linear(in_features=512, out_features=2048, bias=True)\n",
       "          (gelu): QuickGELU()\n",
       "          (c_proj): Linear(in_features=2048, out_features=512, bias=True)\n",
       "        )\n",
       "        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "      )\n",
       "      (9): ResidualAttentionBlock(\n",
       "        (attn): MultiheadAttention(\n",
       "          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n",
       "        )\n",
       "        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): Sequential(\n",
       "          (c_fc): Linear(in_features=512, out_features=2048, bias=True)\n",
       "          (gelu): QuickGELU()\n",
       "          (c_proj): Linear(in_features=2048, out_features=512, bias=True)\n",
       "        )\n",
       "        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "      )\n",
       "      (10): ResidualAttentionBlock(\n",
       "        (attn): MultiheadAttention(\n",
       "          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n",
       "        )\n",
       "        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): Sequential(\n",
       "          (c_fc): Linear(in_features=512, out_features=2048, bias=True)\n",
       "          (gelu): QuickGELU()\n",
       "          (c_proj): Linear(in_features=2048, out_features=512, bias=True)\n",
       "        )\n",
       "        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "      )\n",
       "      (11): ResidualAttentionBlock(\n",
       "        (attn): MultiheadAttention(\n",
       "          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)\n",
       "        )\n",
       "        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): Sequential(\n",
       "          (c_fc): Linear(in_features=512, out_features=2048, bias=True)\n",
       "          (gelu): QuickGELU()\n",
       "          (c_proj): Linear(in_features=2048, out_features=512, bias=True)\n",
       "        )\n",
       "        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (token_embedding): Embedding(49408, 512)\n",
       "  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       ")"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model, preprocess = clip.load(\"ViT-B/32\")\n",
    "\n",
    "\n",
    "model.cuda().eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "d346d78d",
   "metadata": {},
   "outputs": [],
   "source": [
    "device = \"cuda\" if torch.cuda.is_available() else \"cpu\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "ee4aa65c",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "3413 3413\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 3413/3413 [03:39<00:00, 15.56it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sym diag:3413\n",
      "1\n",
      "4043 4043\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 4043/4043 [03:50<00:00, 17.53it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sym diag:4043\n",
      "2\n",
      "3226 3226\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 3226/3226 [02:59<00:00, 17.96it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sym diag:3226\n",
      "3\n",
      "3275 3275\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 3275/3275 [03:07<00:00, 17.51it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sym diag:3275\n",
      "4\n",
      "2998 2998\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2998/2998 [02:38<00:00, 18.89it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sym diag:2998\n",
      "5\n",
      "3081 3081\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 3081/3081 [02:42<00:00, 18.92it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sym diag:3081\n",
      "6\n",
      "4749 4749\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 4749/4749 [02:36<00:00, 30.29it/s] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sym diag:4749\n",
      "7\n",
      "2363 2363\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2363/2363 [01:49<00:00, 21.58it/s] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sym diag:2363\n",
      "8\n",
      "2856 2856\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2856/2856 [02:50<00:00, 16.73it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sym diag:2856\n",
      "9\n",
      "2335 2335\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2335/2335 [01:44<00:00, 22.35it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sym diag:2335\n",
      "10\n",
      "1677 1677\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1677/1677 [01:08<00:00, 24.40it/s] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sym diag:1677\n",
      "11\n",
      "2593 2593\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2593/2593 [02:07<00:00, 20.34it/s] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sym diag:2593\n",
      "12\n",
      "1841 1841\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1841/1841 [01:36<00:00, 19.04it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sym diag:1841\n",
      "13\n",
      "1639 1639\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1639/1639 [01:04<00:00, 25.37it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sym diag:1639\n",
      "14\n",
      "875 875\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 875/875 [00:22<00:00, 38.65it/s] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sym diag:875\n",
      "15\n",
      "1066 1066\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1066/1066 [00:48<00:00, 22.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sym diag:1066\n",
      "16\n",
      "932 932\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 932/932 [00:39<00:00, 23.33it/s] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sym diag:932\n",
      "17\n",
      "588 588\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 588/588 [00:20<00:00, 28.15it/s] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sym diag:588\n",
      "18\n",
      "671 671\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 671/671 [00:14<00:00, 47.07it/s] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sym diag:671\n",
      "19\n",
      "239 239\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 239/239 [00:11<00:00, 19.96it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sym diag:239\n",
      "20\n",
      "362 362\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 362/362 [00:12<00:00, 29.46it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sym diag:362\n",
      "21\n",
      "438 438\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 438/438 [00:14<00:00, 29.71it/s] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sym diag:438\n",
      "22\n",
      "528 528\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 528/528 [00:21<00:00, 25.07it/s] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sym diag:528\n",
      "23\n",
      "254 254\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 254/254 [00:04<00:00, 59.11it/s] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sym diag:254\n",
      "24\n",
      "50 50\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 50/50 [00:01<00:00, 27.08it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sym diag:50\n",
      "25\n",
      "27 27\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 27/27 [00:01<00:00, 21.86it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sym diag:27\n",
      "26\n",
      "14 14\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 14/14 [00:00<00:00, 50.09it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sym diag:14\n",
      "27\n",
      "0 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0it [00:00, ?it/s]\n"
     ]
    },
    {
     "ename": "RuntimeError",
     "evalue": "stack expects a non-empty TensorList",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_700750/3573580038.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     32\u001b[0m             \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"{image_path} corrupted\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     33\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 34\u001b[0;31m     \u001b[0mimages_prep\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimages_prep\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     35\u001b[0m     \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mempty_cache\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     36\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: stack expects a non-empty TensorList"
     ]
    }
   ],
   "source": [
    "import tqdm\n",
    "from pathlib import Path\n",
    "\n",
    "# processed_images = [path_processed_images + \"/\" + img_id + \".npy\" for img_id in data['id']]\n",
    "image_names = list(dict_topic2img.keys())\n",
    "count = 10\n",
    "end = len(image_names) // count + 1\n",
    "similarity_CLIP = []\n",
    "\n",
    "total_proc_imgs = 0\n",
    "for k in range(0, end):\n",
    "    print(k)                    \n",
    "    imgs_corrupted = []\n",
    "    images_prep = []\n",
    "    \n",
    "    texts = image_names[count*k:count*(k+1)]\n",
    "    image_ids = [list(dict_topic2img[topic]) for topic in texts]\n",
    "    texts_k = [[topic] * len(list(dict_topic2img[topic])) for topic in texts]\n",
    "#     print(len(texts_k), len(image_ids))\n",
    "    flat_image_ids = [item for sublist in image_ids for item in sublist]\n",
    "    flat_texts = [item for sublist in texts_k for item in sublist]\n",
    "    texts = flat_texts\n",
    "    k_processed_images = [path_processed_images + \"/\" + img_id + \".npy\" for img_id in flat_image_ids]\n",
    "    print(len(texts), len(k_processed_images))\n",
    "#     break\n",
    "    \n",
    "    for image_path in tqdm.tqdm(k_processed_images):\n",
    "        if Path(image_path).is_file():         \n",
    "            images_prep.append(torch.from_numpy(np.load(image_path)).to(device))\n",
    "        else:\n",
    "            imgs_corrupted.append(image_path)\n",
    "            print(f\"{image_path} corrupted\")\n",
    "            \n",
    "    images_prep = torch.stack(images_prep)\n",
    "    torch.cuda.empty_cache()\n",
    "                    \n",
    "    with torch.no_grad():\n",
    "        image_features = model.encode_image(images_prep)\n",
    "        image_features /= image_features.norm(dim=-1, keepdim=True)\n",
    "\n",
    "#         text_tokens = clip.tokenize([desc for desc in texts]).to(device)\n",
    "        text_tokens = clip.tokenize([f\"a photo of a {desc}\" for desc in texts]).to(device)\n",
    "        text_features = model.encode_text(text_tokens).float()\n",
    "        text_features /= text_features.norm(dim=-1, keepdim=True)\n",
    "                    \n",
    "    similarity = text_features.cpu().numpy() @ image_features.cpu().numpy().T # dot product of image and text features\n",
    "    similarity_diag = torch.from_numpy(similarity.diagonal())\n",
    "    \n",
    "    similarity_CLIP.append(similarity_diag)\n",
    "    print(f\"sym diag:{len(similarity_diag)}\")\n",
    "# print(total_proc_imgs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "8f3fa909",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(46133,)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "similarity_CLIP2 = torch.hstack(similarity_CLIP).cpu().numpy()\n",
    "similarity_CLIP2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "21dd899d",
   "metadata": {},
   "outputs": [],
   "source": [
    "dict_img2income = {}\n",
    "\n",
    "for income, image_id in zip(data['income'], data['id']):\n",
    "    dict_img2income[image_id] = income"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "86d75a39",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>topics</th>\n",
       "      <th>income</th>\n",
       "      <th>CLIP score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5d4be7f4cf0b3a0f3f34656f</td>\n",
       "      <td>family snapshots</td>\n",
       "      <td>738.000000</td>\n",
       "      <td>0.302874</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5ec4f913f0611d7ddd74143c</td>\n",
       "      <td>family snapshots</td>\n",
       "      <td>3195.000000</td>\n",
       "      <td>0.274862</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5d4be8bbcf0b3a0f3f3474d2</td>\n",
       "      <td>family snapshots</td>\n",
       "      <td>311.104038</td>\n",
       "      <td>0.217679</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5d4be94fcf0b3a0f3f34862a</td>\n",
       "      <td>family snapshots</td>\n",
       "      <td>694.000000</td>\n",
       "      <td>0.215334</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5d4be0aacf0b3a0f3f339a57</td>\n",
       "      <td>family snapshots</td>\n",
       "      <td>5621.000000</td>\n",
       "      <td>0.275776</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46128</th>\n",
       "      <td>5d4be620cf0b3a0f3f3431a9</td>\n",
       "      <td>baking sheets</td>\n",
       "      <td>3755.000000</td>\n",
       "      <td>0.280414</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46129</th>\n",
       "      <td>5d4be623cf0b3a0f3f3431fd</td>\n",
       "      <td>baking tables</td>\n",
       "      <td>3755.000000</td>\n",
       "      <td>0.238887</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46130</th>\n",
       "      <td>5d4be624cf0b3a0f3f343213</td>\n",
       "      <td>electricity wires</td>\n",
       "      <td>3755.000000</td>\n",
       "      <td>0.237936</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46131</th>\n",
       "      <td>5d4be82dcf0b3a0f3f346b93</td>\n",
       "      <td>meat markets</td>\n",
       "      <td>4608.946297</td>\n",
       "      <td>0.292274</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46132</th>\n",
       "      <td>5d4be838cf0b3a0f3f346ce5</td>\n",
       "      <td>vegetable markets</td>\n",
       "      <td>4608.946297</td>\n",
       "      <td>0.304926</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>46133 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                             id             topics       income  CLIP score\n",
       "0      5d4be7f4cf0b3a0f3f34656f   family snapshots   738.000000    0.302874\n",
       "1      5ec4f913f0611d7ddd74143c   family snapshots  3195.000000    0.274862\n",
       "2      5d4be8bbcf0b3a0f3f3474d2   family snapshots   311.104038    0.217679\n",
       "3      5d4be94fcf0b3a0f3f34862a   family snapshots   694.000000    0.215334\n",
       "4      5d4be0aacf0b3a0f3f339a57   family snapshots  5621.000000    0.275776\n",
       "...                         ...                ...          ...         ...\n",
       "46128  5d4be620cf0b3a0f3f3431a9      baking sheets  3755.000000    0.280414\n",
       "46129  5d4be623cf0b3a0f3f3431fd      baking tables  3755.000000    0.238887\n",
       "46130  5d4be624cf0b3a0f3f343213  electricity wires  3755.000000    0.237936\n",
       "46131  5d4be82dcf0b3a0f3f346b93       meat markets  4608.946297    0.292274\n",
       "46132  5d4be838cf0b3a0f3f346ce5  vegetable markets  4608.946297    0.304926\n",
       "\n",
       "[46133 rows x 4 columns]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# save CLIP scores to dataframe ..\n",
    "# results_df = data\n",
    "# results_df[\"CLIP score\"] = similarity_CLIP2\n",
    "\n",
    "image_names = list(dict_topic2img.keys())\n",
    "count = 10\n",
    "end = len(image_names) // count + 1\n",
    "similarity_CLIP = []\n",
    "\n",
    "results_df = pd.DataFrame()\n",
    "\n",
    "ids = []\n",
    "topics = []\n",
    "income = []\n",
    "for k in range(0, end):\n",
    "    imgs_corrupted = []\n",
    "    images_prep = []\n",
    "    \n",
    "    texts = image_names[count*k:count*(k+1)]\n",
    "    image_ids = [list(dict_topic2img[topic]) for topic in texts]\n",
    "    texts_k = [[topic] * len(list(dict_topic2img[topic])) for topic in texts]\n",
    "    flat_image_ids = [item for sublist in image_ids for item in sublist]\n",
    "    flat_texts = [item for sublist in texts_k for item in sublist]\n",
    "    for image_id in flat_image_ids:\n",
    "        ids.append(image_id)\n",
    "        income.append(dict_img2income[image_id])\n",
    "    \n",
    "    for topic in flat_texts:\n",
    "        topics.append(topic)\n",
    "    \n",
    "results_df['id'] = ids\n",
    "results_df['topics'] = topics\n",
    "results_df['income'] = income\n",
    "results_df['CLIP score'] = similarity_CLIP2\n",
    "\n",
    "results_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "46ad50e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df = pd.merge(results_df, data[['id','region.id', 'country.name','imageRelPath']], on='id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "6b39571d",
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df.to_csv(\"data/results_CLIP_sep.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "12202c77",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2a2e5257",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}