Data_preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib as mp\n",
    "import numpy as np\n",
    "import json\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def readTextFile(path):\n",
    "    with open(path,encoding=\"utf8\") as f:\n",
    "        caption = f.read()\n",
    "    return caption"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "captions=readTextFile(\"captions.txt\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "captions= captions.split('\\n')[1:-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40455\n"
     ]
    }
   ],
   "source": [
    "print(len(captions))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def spliting(c):\n",
    "    x=c.split('.jpg,',1)\n",
    "    image=x[0]\n",
    "    caption=x[1]\n",
    "    return image,caption"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "image_caption={}\n",
    "for i in captions:\n",
    "    image,caption=spliting(i)\n",
    "    if(image_caption.get(image) is None):\n",
    "        image_caption[image]=[]\n",
    "    image_caption[image].append(caption)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['A girl is stretched out in shallow water',\n",
       " 'A girl wearing a red and multi-colored bikini is laying on her back in shallow water .',\n",
       " 'A little girl in a red swimsuit is laying on her back in shallow water .',\n",
       " '\"A young girl is lying in the sand , while ocean water is surrounding her .\"',\n",
       " 'Girl wearing a bikini lying on her back in a shallow pool of clear blue water .']"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "image_caption['667626_18933d713e']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#data cleaning\n",
    "def clean_caption(caption):\n",
    "    caption=caption.lower()\n",
    "    caption=re.sub('[^a-z]+',\" \",caption)\n",
    "    caption=re.sub('^an',\" \",caption)\n",
    "    caption=caption.split()\n",
    "    caption=[s for s in caption if len(s)>1]\n",
    "    caption=\" \".join(caption)\n",
    "    return caption\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'cat is sitting over the house'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clean_caption(\"An cat is sitting over the house # 64\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "for key,caption in image_caption.items():\n",
    "    for i in range(len(caption)):\n",
    "        caption[i]=clean_caption(caption[i])\n",
    "        \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['girl is stretched out in shallow water',\n",
       " 'girl wearing red and multi colored bikini is laying on her back in shallow water',\n",
       " 'little girl in red swimsuit is laying on her back in shallow water',\n",
       " 'young girl is lying in the sand while ocean water is surrounding her',\n",
       " 'girl wearing bikini lying on her back in shallow pool of clear blue water']"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "image_caption[\"667626_18933d713e\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "\n",
    "with open('file.txt', 'w') as file:\n",
    "     file.write(json.dumps(image_caption))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}