prepare_caffe_and_dictionary_coco.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare model for CoCo\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Setup \n",
    "\n",
    "originalImagesPath = 'data/coco/originalImages'\n",
    "preprocessedImagesPath = 'data/coco/processedImages/'\n",
    "\n",
    "caffe_root = '/home/intuinno/codegit/caffe/'\n",
    "\n",
    "vgg_ilsvrc_19_layoutFileName = caffe_root + 'models/vgg_ilsvrc_19/VGG_ILSVRC_19_layers_deploy.prototxt'\n",
    "vgg_ilsvrc_19_modelFileName = caffe_root + 'models/vgg_ilsvrc_19/VGG_ILSVRC_19_layers.caffemodel'\n",
    "\n",
    "dataPath = 'data/coco/'\n",
    "annotation_path = dataPath + 'annotations/captions_train2014.json'\n",
    "splitFileName = dataPath + 'dataset_coco.json'\n",
    "\n",
    "experimentPrefix = '.exp1'\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Import\n",
    "\n",
    "import pdb\n",
    "from sys import stdout\n",
    "import scipy\n",
    "import  cPickle as pickle\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "import sys\n",
    "sys.path.insert(0, caffe_root + 'python')\n",
    "\n",
    "import caffe\n",
    "\n",
    "plt.rcParams['figure.figsize'] = (10, 10)\n",
    "plt.rcParams['image.interpolation'] = 'nearest'\n",
    "plt.rcParams['image.cmap'] = 'gray'\n",
    "\n",
    "import os\n",
    "\n",
    "import pandas as pd\n",
    "import nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "caffe.set_device(1)\n",
    "caffe.set_mode_gpu()\n",
    "\n",
    "net = caffe.Net(vgg_ilsvrc_19_layoutFileName,\n",
    "                vgg_ilsvrc_19_modelFileName,\n",
    "                caffe.TEST)\n",
    "\n",
    "# input preprocessing: 'data' is the name of the input blob == net.inputs[0]\n",
    "transformer = caffe.io.Transformer({'data': net.blobs['data'].data.shape})\n",
    "transformer.set_transpose('data', (2,0,1))\n",
    "transformer.set_mean('data', np.load(caffe_root + 'python/caffe/imagenet/ilsvrc_2012_mean.npy').mean(1).mean(1)) # mean pixel\n",
    "transformer.set_raw_scale('data', 255)  # the reference model operates on images in [0,255] range instead of [0,1]\n",
    "transformer.set_channel_swap('data', (2,1,0))  # the reference model has channels in BGR order instead of RGB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Create file list \n",
    "# coco.devImages.txt \n",
    "# coco.trainImages.txt \n",
    "# coco.testImages.txt\n",
    "\n",
    "import json\n",
    "from pprint import pprint\n",
    "\n",
    "\n",
    "with open(splitFileName) as f:\n",
    "    data = json.load(f)\n",
    "\n",
    "df = pd.DataFrame(data['images'])\n",
    "\n",
    "files = [ 'dev','test','train']\n",
    "\n",
    "dataDict = {}\n",
    "\n",
    "dataDict['dev'] = df[df.split == 'val']\n",
    "dataDict['test'] = df[df.split == 'test']\n",
    "dataDict['restval'] = df[df.split == 'restval']\n",
    "dataDict['train'] = df[df.split == 'train']\n",
    "\n",
    "for f in files:\n",
    "    dataDict[f]['filename'].to_csv(dataPath + 'coco.' + f + 'Images.txt',index=False)\n",
    "    \n",
    "\n",
    "def buildCapDict(sentences):\n",
    "    return [s[u'raw'] for s in sentences ]\n",
    "\n",
    "df['captions'] = df.apply(lambda row: buildCapDict(row['sentences']), axis=1)\n",
    "\n",
    "capDict = df.loc[:,['filename', 'captions']].set_index('filename').to_dict()\n",
    "\n",
    "capDict = capDict['captions']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Let's build dictionary\n",
    "\n",
    "# Let's make dictionary\n",
    "\n",
    "corpus = df['captions'].values\n",
    "corpus2 = [' '.join(c) for c in corpus]\n",
    "corpus3 = ' '.join(corpus2)\n",
    "\n",
    "words = nltk.FreqDist(corpus3.split()).most_common()\n",
    "\n",
    "wordsDict = {words[i][0]:i+2 for i in range(len(words))}\n",
    "\n",
    "with open(dataPath + 'dictionary.pkl', 'wb') as f:\n",
    "    pickle.dump(wordsDict, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(u'a', 660420),\n",
       " (u'A', 359312),\n",
       " (u'on', 221008),\n",
       " (u'of', 211224),\n",
       " (u'in', 189904),\n",
       " (u'the', 177513),\n",
       " (u'with', 160057),\n",
       " (u'and', 145571),\n",
       " (u'is', 100190),\n",
       " (u'man', 71303)]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dev\n",
      "5000(5000, 100352)\n",
      "test\n",
      "5000(5000, 100352)\n",
      "train\n",
      "82783(82783, 100352)\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'coco_feature' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-16-9812ddc91edd>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     43\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     44\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mfname\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m'train'\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 45\u001b[1;33m         \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msavez\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdataPath\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mcoco_feature\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mfname\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mexperimentPrefix\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mresultSave32\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mindices\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mresultSave32\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mindices\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mindptr\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mresultSave32\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mindptr\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mshape\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mresultSave\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     46\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     47\u001b[0m         \u001b[0mfileName\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdataPath\u001b[0m \u001b[1;33m+\u001b[0m \u001b[1;34m'coco_feature.'\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mfname\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mexperimentPrefix\u001b[0m \u001b[1;33m+\u001b[0m \u001b[1;34m'.pkl'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m'wb'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'coco_feature' is not defined"
     ]
    }
   ],
   "source": [
    "# set net to batch size of 50\n",
    "# net.blobs['data'].reshape(10,3,224,224)\n",
    "\n",
    "# resultingFeatures = \n",
    "\n",
    "# files = ['test', 'train', 'dev']\n",
    "files = [ 'dev','test','train']\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "for fname in files:\n",
    "    print fname \n",
    "    f = open(dataPath + 'coco.' + fname + 'Images.txt')\n",
    "    counter = 0\n",
    "    \n",
    "    imageList = [i for i in f]\n",
    "    numImage = len(imageList)\n",
    "#     pdb.set_trace()\n",
    "    \n",
    "    result = np.empty((numImage, 100352))\n",
    "\n",
    "    for i in range(numImage):\n",
    "        fn = imageList[i].rstrip()\n",
    "        net.blobs['data'].data[...] = transformer.preprocess('data', caffe.io.load_image( preprocessedImagesPath +  fn))\n",
    "        out = net.forward()\n",
    "        feat = net.blobs['conv5_4'].data[0]\n",
    "#         import pdb; pdb.set_trace()\n",
    "    #     print feat.shape\n",
    "#         pdb.set_trace()\n",
    "        reshapeFeat = np.swapaxes(feat, 0,2)\n",
    "        reshapeFeat2 = np.reshape(reshapeFeat,(1,-1))\n",
    "        \n",
    "        counter += 1\n",
    "        stdout.write(\"\\r%d\" % counter)\n",
    "        stdout.flush()\n",
    "        result[i,:] = reshapeFeat2\n",
    "        \n",
    "    print result.shape\n",
    "    \n",
    "    resultSave = scipy.sparse.csr_matrix(result)\n",
    "    resultSave32 = resultSave.astype('float32')\n",
    "    \n",
    "    if fname == 'train':\n",
    "        np.savez(dataPath + 'coco_feature.' + fname + experimentPrefix, data=resultSave32.data, indices=resultSave32.indices, indptr=resultSave32.indptr, shape=resultSave.shape)\n",
    "    else:\n",
    "        fileName = open(dataPath + 'coco_feature.' + fname + experimentPrefix + '.pkl','wb')\n",
    "        pickle.dump(resultSave32, fileName ,-1)\n",
    "        fileName.close()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "np.savez(dataPath + 'coco_feature.' + fname + experimentPrefix, data=resultSave32.data, indices=resultSave32.indices, indptr=resultSave32.indptr, shape=resultSave.shape)\n",
    "  \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def save_sparse_csr(filename,array):\n",
    "    np.savez(filename,data = array.data ,indices=array.indices,\n",
    "             indptr =array.indptr, shape=array.shape )\n",
    "\n",
    "def load_sparse_csr(filename):\n",
    "    loader = np.load(filename)\n",
    "    return csr_matrix((  loader['data'], loader['indices'], loader['indptr']),\n",
    "                         shape = loader['shape'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "files = ['test', 'dev']\n",
    "\n",
    "for name in files:\n",
    "    counter = 0\n",
    "    feat = pickle.load(open(dataPath + 'coco_feature.' + name + experimentPrefix + '.pkl','rb'))\n",
    "    filenames = open(dataPath + 'coco.' + name + 'Images.txt')\n",
    "    cap = []\n",
    "    for imageFile in filenames:\n",
    "        imageFile = imageFile.rstrip()\n",
    "        for sen in capDict[imageFile]:\n",
    "            cap.append([sen.rstrip(), counter])\n",
    "        counter += 1\n",
    "    saveFile = open(dataPath + 'coco_align.' + name + experimentPrefix + '.pkl', 'wb')\n",
    "    pickle.dump(cap, saveFile, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "    pickle.dump(feat, saveFile, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "    saveFile.close()\n",
    "            \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "files = ['train']\n",
    "\n",
    "for name in files:\n",
    "    counter = 0\n",
    "#     feat = pickle.load(open(dataPath + 'coco_feature.' + name +'.pkl','rb'))\n",
    "    filenames = open(dataPath + 'coco.' + name + 'Images.txt')\n",
    "    cap = []\n",
    "    for imageFile in filenames:\n",
    "        imageFile = imageFile.rstrip()\n",
    "        for sen in capDict[imageFile]:\n",
    "            cap.append([sen.rstrip(), counter])\n",
    "        counter += 1\n",
    "    saveFile = open(dataPath + 'coco_align.' + name + experimentPrefix + '.pkl', 'wb')\n",
    "    pickle.dump(cap, saveFile, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "#     pickle.dump(feat, saveFile, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "    saveFile.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "22"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wordsDict['Two']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<82783x100352 sparse matrix of type '<type 'numpy.float32'>'\n",
       "\twith 502170223 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "resultSave32"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "description": "Instant recognition with a pre-trained model and a tour of the net interface for visualizing features and parameters layer-by-layer.",
  "example_name": "Image Classification and Filter Visualization",
  "include_in_docs": true,
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  },
  "priority": 1
 },
 "nbformat": 4,
 "nbformat_minor": 0
}