doc2vec.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Getting Started"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import gensim\n",
    "import os\n",
    "import collections\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_file = \"/home/ubuntu/summarization_query_oriented/data/train_20k.txt\"\n",
    "test_file = \"/home/ubuntu/summarization_query_oriented/data/test_20k_500.txt\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocess Text for Doc2Vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def read_corpus(fname, tokens_only=False):\n",
    "#    with open(fname, encoding=\"iso-8859-1\") as f:\n",
    "    with open(fname) as f:        \n",
    "        for i, line in enumerate(f):\n",
    "            if tokens_only:\n",
    "                yield gensim.utils.simple_preprocess(line, deacc=True)\n",
    "            else:\n",
    "                # For training data, add tags\n",
    "                yield gensim.models.doc2vec.TaggedDocument(gensim.utils.simple_preprocess(line, deacc=True), [i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train_corpus = list(read_corpus(train_file))\n",
    "test_corpus = list(read_corpus(test_file, tokens_only=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[u'bishop', u'of', u'ross', u'ireland', u'pre', u'and', u'post', u'reformation', u'bishops']\n"
     ]
    }
   ],
   "source": [
    "print test_corpus[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "322329\n"
     ]
    }
   ],
   "source": [
    "print len(train_corpus)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "dm = 0\n",
    "min_count = 5\n",
    "window = 10\n",
    "size = 400\n",
    "sample = 1e-4\n",
    "negative = 5\n",
    "workers = 4\n",
    "epoch = 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "model = gensim.models.doc2vec.Doc2Vec(dm=dm,min_count=min_count, window=window, size=size, sample=sample, negative=negative, workers=workers,iter = epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.build_vocab(train_corpus)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training the model (may be long !)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 36min 34s, sys: 30.2 s, total: 37min 4s\n",
      "Wall time: 11min 14s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "179178523"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%time model.train(train_corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model saved\n"
     ]
    }
   ],
   "source": [
    "model_name =\"dm_\"+str(dm)+\"_mc_\"+str(min_count)+\"_w_\"+str(window)+\"_size_\"+str(size)+\"_neg_\"+str(negative)+\"_ep_\"+str(epoch)\n",
    "model.save(\"./models/\"+model_name+\".d2v\")\n",
    "print(\"model saved\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loading it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model loaded\n"
     ]
    }
   ],
   "source": [
    "model_name =\"dm_\"+str(dm)+\"_mc_\"+str(min_count)+\"_w_\"+str(window)+\"_size_\"+str(size)+\"_neg_\"+str(negative)+\"_ep_\"+str(epoch)\n",
    "model_d2v = model.load(\"./models/\"+model_name+\".d2v\")\n",
    "print(\"model loaded\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Testing it !"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([  1.94291547e-02,  -6.70507848e-01,  -2.26440653e-01,\n",
       "        -1.37340426e-01,  -8.35170820e-02,   1.42552927e-01,\n",
       "        -1.29473403e-01,   1.92456543e-01,  -4.53324825e-01,\n",
       "        -1.17267422e-01,   4.75911461e-02,   9.05732960e-02,\n",
       "        -4.17214222e-02,  -3.94426733e-01,   2.02206358e-01,\n",
       "         1.09554358e-01,   1.40604600e-01,   5.75654134e-02,\n",
       "         3.40830684e-01,  -4.91876334e-01,  -3.38415116e-01,\n",
       "         9.42548364e-02,   1.15097314e-02,  -3.56676608e-01,\n",
       "         6.34246230e-01,   1.39211323e-02,  -1.67301565e-01,\n",
       "         2.17303842e-01,   1.49504036e-01,   1.06433123e-01,\n",
       "         1.23297833e-01,   7.93358833e-02,   7.62027428e-02,\n",
       "         3.01561892e-01,  -5.39335430e-01,  -1.75229162e-02,\n",
       "        -5.59485657e-03,   1.30450845e-01,   5.04945330e-02,\n",
       "         3.42386849e-02,  -9.35370177e-02,  -1.54827684e-01,\n",
       "        -1.75499935e-02,   1.13537386e-01,  -2.83598959e-01,\n",
       "        -5.21475673e-02,   2.95050889e-01,  -6.58204556e-01,\n",
       "         2.75894493e-01,  -1.72511637e-01,   2.17648223e-01,\n",
       "         1.80814803e-01,  -2.90220737e-01,   1.27050383e-02,\n",
       "         7.29964226e-02,   9.13439915e-02,  -1.39631256e-01,\n",
       "         2.84213573e-01,  -1.03506304e-01,  -1.82170674e-01,\n",
       "        -1.32375121e-01,   2.40783855e-01,   9.09092724e-02,\n",
       "        -7.07444176e-02,  -6.23599952e-03,  -2.39993408e-02,\n",
       "         3.84144820e-02,   2.02108756e-01,   2.27865592e-01,\n",
       "        -1.48480490e-01,   1.79977313e-01,  -4.49590057e-01,\n",
       "         5.91712296e-02,  -2.56907970e-01,  -2.87397765e-02,\n",
       "         1.78754758e-02,   3.13262135e-01,  -4.51949378e-03,\n",
       "         1.76173925e-01,   1.06528118e-01,  -3.39281321e-01,\n",
       "         9.23840702e-02,   5.68939447e-01,  -1.40590981e-01,\n",
       "         1.08506326e-02,  -6.00626804e-02,  -3.61506313e-01,\n",
       "        -1.86749876e-01,  -2.74689972e-01,  -7.61571676e-02,\n",
       "         3.71831566e-01,  -4.44794863e-01,   3.86302441e-01,\n",
       "        -6.28631949e-01,   1.06114700e-01,  -1.98870778e-01,\n",
       "        -1.62388712e-01,   4.27876502e-01,   1.68502748e-01,\n",
       "        -2.01479524e-01,   9.43354219e-02,  -2.21851274e-01,\n",
       "        -7.61542320e-02,  -2.86667883e-01,   9.89427119e-02,\n",
       "        -3.16397041e-01,  -2.53487140e-01,   2.39782557e-02,\n",
       "        -3.84572923e-01,  -4.57626224e-01,   2.15024389e-02,\n",
       "         1.76295668e-01,  -1.57447144e-01,  -4.17060554e-01,\n",
       "         1.95656106e-01,  -4.05048996e-01,  -5.10811470e-02,\n",
       "         3.16959530e-01,   5.15979640e-02,   2.57814825e-01,\n",
       "        -1.87683240e-01,  -8.40605944e-02,  -2.40465831e-02,\n",
       "        -3.71519327e-01,   1.23568811e-01,   1.02154329e-01,\n",
       "         6.60816804e-02,   5.89106500e-01,  -7.78236389e-01,\n",
       "         2.79011708e-02,   3.40851545e-02,  -3.69574726e-01,\n",
       "        -4.16374616e-02,  -3.91446203e-01,   1.53124854e-01,\n",
       "        -3.24391454e-01,   3.48815829e-01,   8.80271383e-03,\n",
       "         9.03765783e-02,  -9.94270369e-02,   2.81968892e-01,\n",
       "        -1.74867406e-01,   1.39367551e-01,  -1.17417231e-01,\n",
       "         1.37876213e-01,   5.44578768e-02,  -2.15564087e-01,\n",
       "        -1.84170157e-01,  -4.75116074e-01,  -3.86062890e-01,\n",
       "        -1.43369555e-01,   1.48600906e-01,   7.71587491e-02,\n",
       "         1.97255254e-01,   9.24606528e-03,   7.42970407e-02,\n",
       "        -8.41066837e-02,   1.31458104e-01,   2.19417349e-01,\n",
       "         3.40125084e-01,   3.23817641e-01,  -2.79115289e-01,\n",
       "        -4.81358141e-01,   2.37459138e-01,   1.46854267e-01,\n",
       "        -5.80625981e-02,   1.10700369e-01,   7.59170651e-02,\n",
       "         1.51070118e-01,  -2.71257609e-01,  -5.41302264e-01,\n",
       "        -2.80504644e-01,   1.02673113e-01,  -4.27386649e-02,\n",
       "        -4.11388904e-01,   8.38212147e-02,  -1.30947813e-01,\n",
       "         4.82478231e-01,  -1.58308849e-01,   1.17044551e-02,\n",
       "        -1.06908403e-01,  -3.90544742e-01,  -3.77912402e-01,\n",
       "         1.18913054e-01,   5.10572046e-02,   2.74473339e-01,\n",
       "        -1.00171961e-01,   3.39570679e-02,   1.27610311e-01,\n",
       "        -4.41718042e-01,  -1.64305240e-01,   1.62938774e-01,\n",
       "        -2.99929082e-01,   2.64124393e-01,  -1.06825896e-01,\n",
       "        -9.23115313e-02,  -3.06565762e-02,   4.94459331e-01,\n",
       "        -5.25260091e-01,  -4.41227734e-01,  -1.77387848e-01,\n",
       "        -6.54564053e-02,   5.41369691e-02,   2.51281470e-01,\n",
       "        -8.16594362e-02,   1.27540201e-01,  -7.70713985e-02,\n",
       "        -4.30547632e-02,  -3.62934351e-01,   6.15729345e-03,\n",
       "         8.90613273e-02,  -5.25662117e-03,  -3.58981490e-01,\n",
       "         1.34463519e-01,   1.59784064e-01,   3.44189793e-01,\n",
       "         1.00691520e-01,  -1.60611749e-01,  -1.12129621e-01,\n",
       "         1.19938165e-01,  -1.66146129e-01,   3.36875200e-01,\n",
       "         4.42008823e-02,   2.45627135e-01,  -3.63297984e-02,\n",
       "         8.48849565e-02,   1.88627481e-01,   1.61514744e-01,\n",
       "        -2.25736007e-01,  -2.24451795e-01,  -1.14574663e-01,\n",
       "        -6.29855394e-02,   2.33048469e-01,   3.86743695e-01,\n",
       "         3.18491161e-01,  -4.28027898e-01,   1.76863849e-01,\n",
       "        -3.11704695e-01,   6.14701100e-02,  -1.27455378e-02,\n",
       "        -1.64951429e-01,  -3.97933275e-01,   1.68038994e-01,\n",
       "        -6.87360168e-02,  -9.86284204e-03,   8.92082136e-03,\n",
       "        -5.24616428e-02,  -2.55427092e-01,  -3.83891732e-01,\n",
       "        -1.78576648e-01,   2.57193565e-01,  -6.02884777e-02,\n",
       "         1.49934143e-01,  -2.69245505e-01,   6.22000769e-02,\n",
       "         1.35050088e-01,   1.81360930e-01,  -7.11747631e-02,\n",
       "         9.33905914e-02,  -3.57305974e-01,  -3.53994101e-01,\n",
       "         3.34481299e-01,  -2.70496786e-01,  -1.20980114e-01,\n",
       "         1.67733014e-01,  -8.99212360e-02,   3.74671221e-01,\n",
       "        -1.43251091e-01,   2.52979994e-01,  -3.94176431e-02,\n",
       "         1.33688189e-02,   4.69096787e-02,   1.56851225e-02,\n",
       "         9.84819382e-02,  -1.58452228e-01,  -2.49591261e-01,\n",
       "         2.48677343e-01,  -8.76636729e-02,   4.73482013e-01,\n",
       "         1.63244784e-01,  -2.82376502e-02,   1.83682427e-01,\n",
       "         4.68411595e-02,  -1.43640012e-01,  -2.05561414e-01,\n",
       "        -4.53055725e-02,  -3.37874323e-01,   2.38926470e-01,\n",
       "         3.20658572e-02,   5.88992774e-01,  -8.48794132e-02,\n",
       "        -1.55667871e-01,   1.56964689e-01,  -1.31106079e-01,\n",
       "         1.52556330e-01,   2.24528834e-02,   2.78628264e-02,\n",
       "         1.35383196e-02,  -1.59447685e-01,  -1.96471140e-01,\n",
       "        -3.04159582e-01,   3.15503664e-02,   4.05375183e-01,\n",
       "         2.23127395e-01,   3.24717611e-02,   1.13645360e-01,\n",
       "         9.66959298e-02,  -6.15557469e-02,  -8.63678604e-02,\n",
       "        -4.22812700e-01,  -4.76660691e-02,   6.03731513e-01,\n",
       "        -3.78370047e-01,   9.82453004e-02,   3.29256058e-01,\n",
       "         4.69208956e-02,   2.43061870e-01,   2.87769660e-02,\n",
       "        -1.42806143e-01,   5.84369376e-02,   4.29362357e-01,\n",
       "        -6.10081673e-01,  -6.02253199e-01,  -1.20071694e-01,\n",
       "        -2.16085926e-01,   1.94598451e-01,  -2.62739718e-01,\n",
       "         7.15447888e-02,  -7.90075511e-02,  -6.10435128e-01,\n",
       "         1.63048849e-01,  -1.85996786e-01,   1.89823568e-01,\n",
       "        -5.08393277e-04,   1.45529360e-01,  -1.27555072e-01,\n",
       "        -2.13292256e-01,   3.41971368e-01,  -3.11447978e-01,\n",
       "        -2.42582217e-01,  -7.24796832e-01,  -2.53088009e-02,\n",
       "        -1.22723252e-01,  -5.43204486e-01,   1.22198589e-01,\n",
       "         2.95642531e-04,  -2.88046241e-01,  -2.75476545e-01,\n",
       "        -2.06853133e-02,   2.13633627e-02,  -9.56457555e-02,\n",
       "         1.77702188e-01,  -2.11467698e-01,  -1.92402855e-01,\n",
       "         3.74773532e-01,  -1.68580040e-01,   6.98875725e-01,\n",
       "         7.10106194e-02,  -2.59847462e-01,  -4.90684882e-02,\n",
       "         5.06878495e-02,  -4.08769011e-01,  -4.10627685e-02,\n",
       "         1.27455015e-02,   2.35072467e-02,  -1.58985764e-01,\n",
       "         1.43116251e-01,   4.88405302e-02,  -3.68938804e-01,\n",
       "         1.14638589e-01,  -1.63277183e-02,  -4.41510290e-01,\n",
       "        -6.41507804e-02,  -3.65888238e-01,   4.10705119e-01,\n",
       "        -2.60805935e-01,   3.87757807e-03,  -8.42808709e-02,\n",
       "        -1.57844424e-01,   9.16289330e-01,   6.50516510e-01,\n",
       "        -3.09374910e-02,   2.65212238e-01,  -2.56139606e-01,\n",
       "        -3.06136739e-02,   2.79309154e-01,  -9.45874862e-03,\n",
       "         2.45188892e-01,   7.93372467e-02,   3.85511786e-01,\n",
       "         1.83743238e-01,  -6.69465736e-02,   1.16138473e-01,\n",
       "        -9.51830521e-02,   1.86729595e-01,  -1.83225751e-01,\n",
       "        -9.03959274e-02,  -1.35720387e-01,   9.58318785e-02,\n",
       "         3.00081242e-02], dtype=float32)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_d2v.infer_vector(['only', 'you', 'can', 'prevent', 'forrest', 'fires'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "ranks = []\n",
    "second_ranks = []\n",
    "for doc_id in range(100):\n",
    "    inferred_vector = model.infer_vector(train_corpus[doc_id].words)\n",
    "    sims = model.docvecs.most_similar([inferred_vector], topn=len(model.docvecs))\n",
    "    rank = [docid for docid, sim in sims].index(doc_id)\n",
    "    ranks.append(rank)\n",
    "    \n",
    "    second_ranks.append(sims[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Document (55): «talks about people stomachs»\n",
      "\n",
      "SIMILAR/DISSIMILAR DOCS PER MODEL Doc2Vec(dbow,d400,n5,hs,mc5,s0.0001,t4):\n",
      "\n",
      "MOST (55564, 0.22469821572303772): «gratian is assassinated and constantine iii general magister militum declares himself roman emperor to extend his dominion over gaul and spain he takes practically all the roman garrisons from britain and crosses the english channel constantine occupies arles and established tenuous authority over gaul sharing control with marauding barbarians this is generally seen as the beginning of rome withdrawal from britain»\n",
      "\n",
      "MEDIAN (166894, -5.2908435463905334e-05): «beechcraft queen air»\n",
      "\n",
      "LEAST (210436, -0.21172496676445007): «with no constitutional amendment done annadurai declared january the th republic day of india and also the day the constitution which in essence enshrined hindi as the official language of india came into practice as day of mourning this move was opposed by the then chief minister of madras state as blasphemous hence annadurai who by then had been trying to shake off the secessionist image of his party declared january as day of mourning he also replaced the slogan of the protests to down with hindi long live the republic nevertheless violence broke out on january initially in madurai which within days spread throughout the state robert hardgrave jr professor of humanities government and asian studies suggests that the elements contributing to the riots were not instigated by dmk or leftists or even the industrialists as the congress government of the state suggested but were genuine frustrations and discontentment which lay beneath the surface of the people of the state»\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Pick a random document from the test corpus and infer a vector from the model\n",
    "doc_id = random.randint(0, 100)\n",
    "inferred_vector = model.infer_vector(test_corpus[doc_id])\n",
    "sims = model.docvecs.most_similar([inferred_vector], topn=len(model.docvecs))\n",
    "\n",
    "# Compare and print the most/median/least similar documents from the train corpus\n",
    "print(u'Test Document ({}): «{}»\\n'.format(doc_id, ' '.join(test_corpus[doc_id])))\n",
    "print(u'SIMILAR/DISSIMILAR DOCS PER MODEL %s:\\n' % model)\n",
    "for label, index in [('MOST', 0), ('MEDIAN', len(sims)//2), ('LEAST', len(sims) - 1)]:\n",
    "    print(u'%s %s: «%s»\\n' % (label, sims[index], ' '.join(train_corpus[sims[index][0]].words)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}