prepare_caffe_and_dictionary_flickr30k.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare model for Flickr8k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Setup \n",
    "\n",
    "originalImagesPath = 'data/flickr30k/originalImages'\n",
    "preprocessedImagesPath = 'data/flickr30k/processedImages/'\n",
    "\n",
    "caffe_root = '/home/intuinno/codegit/caffe/'\n",
    "\n",
    "vgg_ilsvrc_19_layoutFileName = caffe_root + 'models/vgg_ilsvrc_19/VGG_ILSVRC_19_layers_deploy.prototxt'\n",
    "vgg_ilsvrc_19_modelFileName = caffe_root + 'models/vgg_ilsvrc_19/VGG_ILSVRC_19_layers.caffemodel'\n",
    "\n",
    "dataPath = 'data/flickr30k/'\n",
    "\n",
    "annotation_path = dataPath + 'results_20130124.token'\n",
    "\n",
    "numTrain = 25000\n",
    "numTest = 5000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Import\n",
    "\n",
    "import pdb\n",
    "from sys import stdout\n",
    "import scipy\n",
    "import  cPickle as pickle\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "import sys\n",
    "sys.path.insert(0, caffe_root + 'python')\n",
    "\n",
    "import caffe\n",
    "\n",
    "plt.rcParams['figure.figsize'] = (10, 10)\n",
    "plt.rcParams['image.interpolation'] = 'nearest'\n",
    "plt.rcParams['image.cmap'] = 'gray'\n",
    "\n",
    "import os\n",
    "\n",
    "import pandas as pd\n",
    "import nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "caffe.set_device(1)\n",
    "caffe.set_mode_gpu()\n",
    "\n",
    "net = caffe.Net(vgg_ilsvrc_19_layoutFileName,\n",
    "                vgg_ilsvrc_19_modelFileName,\n",
    "                caffe.TEST)\n",
    "\n",
    "# input preprocessing: 'data' is the name of the input blob == net.inputs[0]\n",
    "transformer = caffe.io.Transformer({'data': net.blobs['data'].data.shape})\n",
    "transformer.set_transpose('data', (2,0,1))\n",
    "# transformer.set_mean('data', np.load(caffe_root + 'python/caffe/imagenet/ilsvrc_2012_mean.npy').mean(1).mean(1)) # mean pixel\n",
    "# transformer.set_raw_scale('data', 255)  # the reference model operates on images in [0,255] range instead of [0,1]\n",
    "# transformer.set_channel_swap('data', (2,1,0))  # the reference model has channels in BGR order instead of RGB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dev\n",
      "test\n",
      "train\n"
     ]
    }
   ],
   "source": [
    "from os import listdir\n",
    "from os.path import isfile, join\n",
    "onlyfiles = [f for f in listdir(preprocessedImagesPath) if isfile(join(preprocessedImagesPath, f))]\n",
    "onlyJPGfiles = [f for f in onlyfiles if f.endswith('jpg')]\n",
    "\n",
    "import random\n",
    "random.shuffle(onlyJPGfiles)\n",
    "\n",
    "fileList = {}\n",
    "fileList['train'] = onlyJPGfiles[:numTrain]\n",
    "fileList['test'] = onlyJPGfiles[numTrain:numTrain+numTest]\n",
    "fileList['dev'] = onlyJPGfiles[numTrain+numTest:]\n",
    "\n",
    "files = [ 'dev','test','train']\n",
    "\n",
    "for fname in files:\n",
    "    print fname \n",
    "    f = open(dataPath + 'Flickr_30k.' + fname + 'Images.txt','w')\n",
    "    for item in fileList[fname]:\n",
    "        f.write(\"%s\\n\" % item)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dev\n",
      "1783(1783, 100352)\n",
      "test\n",
      "5000(5000, 100352)\n",
      "train\n",
      "25000(25000, 100352)\n"
     ]
    }
   ],
   "source": [
    "# set net to batch size of 50\n",
    "# net.blobs['data'].reshape(5,3,224,224)\n",
    "\n",
    "# resultingFeatures = \n",
    "\n",
    "# files = ['test', 'train', 'dev']\n",
    "files = [ 'dev','test','train']\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "for fname in files:\n",
    "    print fname \n",
    "    f = open(dataPath + 'Flickr_30k.' + fname + 'Images.txt')\n",
    "    counter = 0\n",
    "    \n",
    "    imageList = [i for i in f]\n",
    "    numImage = len(imageList)\n",
    "#     pdb.set_trace()\n",
    "    \n",
    "    result = np.empty((numImage, 100352))\n",
    "\n",
    "    for i in range(numImage):\n",
    "        fn = imageList[i].rstrip()\n",
    "        net.blobs['data'].data[...] = transformer.preprocess('data', caffe.io.load_image( preprocessedImagesPath +  fn))\n",
    "        out = net.forward()\n",
    "        feat = net.blobs['conv5_4'].data[0]\n",
    "    #     print feat.shape\n",
    "#         pdb.set_trace()\n",
    "        reshapeFeat = np.swapaxes(feat, 0,2)\n",
    "        reshapeFeat2 = np.reshape(reshapeFeat,(1,-1))\n",
    "        \n",
    "        counter += 1\n",
    "        stdout.write(\"\\r%d\" % counter)\n",
    "        stdout.flush()\n",
    "        result[i,:] = reshapeFeat2\n",
    "        \n",
    "    print result.shape\n",
    "    \n",
    "    resultSave = scipy.sparse.csr_matrix(result)\n",
    "    resultSave32 = resultSave.astype('float32')\n",
    "    fileName = open(dataPath + 'flicker_30k_feature.' + fname + '.pkl','wb')\n",
    "    pickle.dump(resultSave32, fileName ,-1)\n",
    "    fileName.close()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "capFile = open(annotation_path)\n",
    "\n",
    "capDict = {}\n",
    "import re\n",
    "for line in capFile:\n",
    "    match = re.search(r'^([\\w]+\\.jpg)#(\\d)\\s([\\w\\W.\\s-]+)$', line)\n",
    "#     print line\n",
    "    if not match:\n",
    "        print line\n",
    "    else:\n",
    "        if match.group(2) == '0':\n",
    "            capDict[match.group(1)] = [match.group(3)]\n",
    "        else:\n",
    "            capDict[match.group(1)].append(match.group(3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "files = ['test', 'train', 'dev']\n",
    "\n",
    "for name in files:\n",
    "    counter = 0\n",
    "    feat = pickle.load(open(dataPath + 'flicker_30k_feature.' + name +'.pkl','rb'))\n",
    "    filenames = open(dataPath + 'Flickr_30k.' + name + 'Images.txt')\n",
    "    cap = []\n",
    "    for imageFile in filenames:\n",
    "        imageFile = imageFile.rstrip()\n",
    "        for sen in capDict[imageFile]:\n",
    "            cap.append([sen.rstrip(), counter])\n",
    "        counter += 1\n",
    "    saveFile = open(dataPath + 'flicker_30k_align.' + name + '.pkl', 'wb')\n",
    "    pickle.dump(cap, saveFile, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "    pickle.dump(feat, saveFile, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "    saveFile.close()\n",
    "            \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Let's make dictionary\n",
    "\n",
    "annotations = pd.read_table(annotation_path, sep='\\t', header=None, names=['image', 'caption'])\n",
    "\n",
    "captions = annotations['caption'].values\n",
    "\n",
    "words = nltk.FreqDist(' '.join(captions).split()).most_common()\n",
    "\n",
    "wordsDict = {words[i][0]:i+2 for i in range(len(words))}\n",
    "\n",
    "with open(dataPath + 'dictionary.pkl', 'wb') as f:\n",
    "    pickle.dump(wordsDict, f)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "17"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wordsDict['Two']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "description": "Instant recognition with a pre-trained model and a tour of the net interface for visualizing features and parameters layer-by-layer.",
  "example_name": "Image Classification and Filter Visualization",
  "include_in_docs": true,
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  },
  "priority": 1
 },
 "nbformat": 4,
 "nbformat_minor": 0
}