alpha_visualization.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#Visualizing the model  \n",
    "This notebook is meant as an example for how to create visualizations\n",
    "like the ones provided in the appendix.\n",
    "\n",
    "It is expected that this might need some slight modification depending on the user's \n",
    "setup. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import theano\n",
    "from theano import tensor\n",
    "from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import cPickle as pkl\n",
    "import numpy\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.cm as cm\n",
    "\n",
    "import skimage\n",
    "import skimage.transform\n",
    "import skimage.io\n",
    "\n",
    "from PIL import Image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import capgen\n",
    "import generate_caps as gencaps\n",
    "import flickr8k\n",
    "import flickr30k\n",
    "import coco"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading the model and dataset  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "datasets = {'flickr8k': (flickr8k.load_data, flickr8k.prepare_data),\n",
    "             'flickr30k': (flickr30k.load_data, flickr30k.prepare_data),\n",
    "             'coco': (coco.load_data, coco.prepare_data)}\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using the following parameters:\n",
      "{'lrate': 0.01, 'decay_c': 0.0, 'patience': 10, 'save_per_epoch': False, 'n_layers_init': 1, 'RL_sumCost': True, 'max_epochs': 5000, 'dispFreq': 100, 'attn_type': 'stochastic', 'alpha_c': 0.0, 'temperature': 1.0, 'n_layers_att': 1, 'saveto': 'model.npz', 'ctx_dim': 512, 'valid_batch_size': 16, 'lstm_encoder': False, 'n_layers_lstm': 1, 'optimizer': 'rmsprop', 'validFreq': 1000, 'dictionary': None, 'batch_size': 16, 'selector': False, 'n_words': 10000, 'dataset': 'flickr30k', 'use_dropout_lstm': False, 'prev2out': False, 'dim': 1000, 'use_dropout': False, 'dim_word': 100, 'sampleFreq': 100, 'semi_sampling_p': 0.5, 'n_layers_out': 1, 'saveFreq': 1000, 'maxlen': 100, 'alpha_entropy_c': 0.002, 'ctx2out': False, 'reload_': False}\n",
      "Loading data\n",
      "... loading data\n"
     ]
    },
    {
     "ename": "IOError",
     "evalue": "[Errno 2] No such file or directory: './flicker_30k_align.train.pkl'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mIOError\u001b[0m                                   Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-6-5d07116ac91f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mcapgen\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mtrain\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'flickr30k'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/Users/anandsampat/Projects/arctic-captions/capgen.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(dim_word, ctx_dim, dim, attn_type, n_layers_att, n_layers_out, n_layers_lstm, n_layers_init, lstm_encoder, prev2out, ctx2out, alpha_entropy_c, RL_sumCost, semi_sampling_p, temperature, patience, max_epochs, dispFreq, decay_c, alpha_c, lrate, selector, n_words, maxlen, optimizer, batch_size, valid_batch_size, saveto, validFreq, saveFreq, sampleFreq, dataset, dictionary, use_dropout, use_dropout_lstm, reload_, save_per_epoch)\u001b[0m\n\u001b[1;32m   1117\u001b[0m     \u001b[0;32mprint\u001b[0m \u001b[0;34m'Loading data'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1118\u001b[0m     \u001b[0mload_data\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprepare_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_dataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1119\u001b[0;31m     \u001b[0mtrain\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mworddict\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mload_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1120\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1121\u001b[0m     \u001b[0;31m# index 0 and 1 always code for the end of sentence and unknown token\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/anandsampat/Projects/arctic-captions/flickr30k.py\u001b[0m in \u001b[0;36mload_data\u001b[0;34m(load_train, load_dev, load_test, path)\u001b[0m\n\u001b[1;32m     66\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     67\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mload_train\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 68\u001b[0;31m         \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;34m'flicker_30k_align.train.pkl'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'rb'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     69\u001b[0m             \u001b[0mtrain_cap\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpkl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     70\u001b[0m             \u001b[0mtrain_feat\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpkl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mIOError\u001b[0m: [Errno 2] No such file or directory: './flicker_30k_align.train.pkl'"
     ]
    }
   ],
   "source": [
    "from capgen import train\n",
    "\n",
    "train(dataset='flickr30k')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading: coco\n"
     ]
    }
   ],
   "source": [
    "# location of the model file, the pkl file should be named \"model_name.npz.pkl\"\n",
    "model= 'model_name.npz'\n",
    "# location of the devset split file like the ones in /splits\n",
    "dev_list = './splits/coco_val.txt' \n",
    "image_path = './path_to_coco_dev_image/'\n",
    "\n",
    "\n",
    "# load model model_options\n",
    "with open('%s.pkl'%model, 'rb') as f:\n",
    "    options = pkl.load(f)\n",
    "\n",
    "print 'Loading: ' + options['dataset']\n",
    "\n",
    "flist = []\n",
    "with open(dev_list, 'r') as f:\n",
    "    for l in f:\n",
    "        flist.append(l.strip())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# keep aspect ratio, and center crop\n",
    "def LoadImage(file_name, resize=256, crop=224):\n",
    "  image = Image.open(file_name)\n",
    "  width, height = image.size\n",
    "\n",
    "  if width > height:\n",
    "    width = (width * resize) / height\n",
    "    height = resize\n",
    "  else:\n",
    "    height = (height * resize) / width\n",
    "    width = resize\n",
    "  left = (width  - crop) / 2\n",
    "  top  = (height - crop) / 2\n",
    "  image_resized = image.resize((width, height), Image.BICUBIC).crop((left, top, left + crop, top + crop))\n",
    "  data = numpy.array(image_resized.convert('RGB').getdata()).reshape(crop, crop, 3)\n",
    "  data = data.astype('float32') / 255\n",
    "  return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "... loading data\n",
      "Data loaded\n"
     ]
    }
   ],
   "source": [
    "    load_data, prepare_data = datasets[options['dataset']]\n",
    "\n",
    "    train, valid, test, worddict = load_data(False, True, False)\n",
    "    print 'Data loaded'\n",
    "\n",
    "    word_idict = dict()\n",
    "    for kk, vv in worddict.iteritems():\n",
    "        word_idict[vv] = kk\n",
    "    word_idict[0] = '<eos>'\n",
    "    word_idict[1] = 'UNK'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating the Theano Graph  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Building f_init... Done\n"
     ]
    }
   ],
   "source": [
    "    # build the sampling functions and model\n",
    "    trng = RandomStreams(1234)\n",
    "    use_noise = theano.shared(numpy.float32(0.), name='use_noise')\n",
    "\n",
    "    params = capgen.init_params(options)\n",
    "    params = capgen.load_params(model, params)\n",
    "    tparams = capgen.init_tparams(params)\n",
    "\n",
    "    # word index\n",
    "    f_init, f_next = capgen.build_sampler(tparams, options, use_noise, trng)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "    trng, use_noise, \\\n",
    "          inps, alphas, alphas_samples, \\\n",
    "          cost, opt_outs = \\\n",
    "          capgen.build_model(tparams, options)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# get the alphas and selector value [called \\beta in the paper]\n",
    "\n",
    "# create update rules for the stochastic attention\n",
    "hard_attn_updates = []\n",
    "if options['attn_type'] == 'stochastic':\n",
    "    baseline_time = theano.shared(numpy.float32(0.), name='baseline_time')\n",
    "    hard_attn_updates += [(baseline_time, baseline_time * 0.9 + 0.1 * opt_outs['masked_cost'].mean())]\n",
    "    hard_attn_updates += opt_outs['attn_updates']\n",
    "    \n",
    "f_alpha = theano.function(inps, alphas, name='f_alpha', updates=hard_attn_updates)\n",
    "if options['selector']:\n",
    "    f_sels = theano.function(inps, opt_outs['selector'], name='f_sels', updates=hard_attn_updates)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generating the Caption and Attention Visualization\n",
    "\n",
    "(The next five cells can be run over and over to visualize a random image)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "idx = numpy.random.randint(0, len(valid[0])) # random image\n",
    "k = 1 # beam width\n",
    "use_gt = False # set to False if you want to use the generated sample\n",
    "gt = valid[0][idx][0] # groundtruth\n",
    "context = numpy.array(valid[1][valid[0][idx][1]].todense()).reshape([14*14, 512]) # annotations\n",
    "img = LoadImage(image_path+flist[valid[0][idx][1]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "if not use_gt:\n",
    "    sample, score = capgen.gen_sample(tparams, f_init, f_next, context, \n",
    "                                      options, trng=trng, k=k, maxlen=200, stochastic=False)\n",
    "    sidx = numpy.argmin(score)\n",
    "    caption = sample[sidx][:-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sample: A bowl of fruit with a bunch of bananas on it .\n",
      "GT: A plate of fruit that includes bananas , pears and apples .\n"
     ]
    }
   ],
   "source": [
    "# print the generated caption and the ground truth\n",
    "if use_gt:\n",
    "    caption = map(lambda w: worddict[w] if worddict[w] < options['n_words'] else 1, gt.split())\n",
    "words = map(lambda w: word_idict[w] if w in word_idict else '<UNK>', caption)\n",
    "print 'Sample:', ' '.join(words)\n",
    "print 'GT:', gt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "alpha = f_alpha(numpy.array(caption).reshape(len(caption),1), \n",
    "                numpy.ones((len(caption),1), dtype='float32'), \n",
    "                context.reshape(1,context.shape[0],context.shape[1]))\n",
    "if options['selector']:\n",
    "    sels = f_sels(numpy.array(caption).reshape(len(caption),1), \n",
    "                   numpy.ones((len(caption),1), dtype='float32'), \n",
    "                   context.reshape(1,context.shape[0],context.shape[1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Displaying the Visualization   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# display the visualization\n",
    "n_words = alpha.shape[0] + 1\n",
    "w = numpy.round(numpy.sqrt(n_words))\n",
    "h = numpy.ceil(numpy.float32(n_words) / w)\n",
    "        \n",
    "plt.subplot(w, h, 1)\n",
    "plt.imshow(img)\n",
    "plt.axis('off')\n",
    "\n",
    "smooth = True\n",
    "\n",
    "for ii in xrange(alpha.shape[0]):\n",
    "    plt.subplot(w, h, ii+2)\n",
    "    lab = words[ii]\n",
    "    if options['selector']:\n",
    "        lab += '(%0.2f)'%sels[ii]\n",
    "    plt.text(0, 1, lab, backgroundcolor='white', fontsize=13)\n",
    "    plt.text(0, 1, lab, color='black', fontsize=13)\n",
    "    plt.imshow(img)\n",
    "    if smooth:\n",
    "        alpha_img = skimage.transform.pyramid_expand(alpha[ii,0,:].reshape(14,14), upscale=16, sigma=20)\n",
    "    else:\n",
    "        alpha_img = skimage.transform.resize(alpha[ii,0,:].reshape(14,14), [img.shape[0], img.shape[1]])\n",
    "    plt.imshow(alpha_img, alpha=0.8)\n",
    "    plt.set_cmap(cm.Greys_r)\n",
    "    plt.axis('off')\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}