PR_HS_LEN_book.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## An unsupervised approach to rank product reviews\n",
    "### Jianwei Wu ; Bing Xu ; Sheng Li ; 2011"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import nltk\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import MHR as mhr\n",
    "import networkx as nx\n",
    "from nltk import word_tokenize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(34754, 16)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def zeroFun(x): return 0\n",
    "dfProducts = pd.read_csv('data/book_sample.csv.gz')\n",
    "dfProducts['pageRank']=dfProducts.apply(zeroFun,axis=1)\n",
    "dfProducts['hits']=dfProducts.apply(zeroFun,axis=1)\n",
    "dfProducts.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def compute_tags(sentence):\n",
    "    tags = []\n",
    "    for token in nltk.pos_tag(nltk.word_tokenize(sentence.lower()), tagset = \"universal\"):\n",
    "        if (token[1] in ['NOUN','VERB','ADJ']):\n",
    "            tags.append(token[0])\n",
    "    return tags\n",
    "\n",
    "def compute_sentence(sentence1, sentence2):\n",
    "    return (len(set(sentence1).intersection(sentence2)) > 0)\n",
    "\n",
    "def compute_sentences_graph(sentences):\n",
    "    sentences_count = len(sentences)\n",
    "    matrix = np.zeros((sentences_count,sentences_count))\n",
    "\n",
    "    for row in range(sentences_count):\n",
    "        for col in range(sentences_count):\n",
    "            matrix[row, col] = compute_sentence(sentences[row],sentences[col])\n",
    "\n",
    "    return matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Run 1 size 48 for 500\n",
      "computing\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python2.7/site-packages/networkx/algorithms/link_analysis/hits_alg.py:198: ComplexWarning: Casting complex values to real discards the imaginary part\n",
      "  hubs=dict(zip(G.nodes(),map(float,h)))\n",
      "/usr/local/lib/python2.7/site-packages/networkx/algorithms/link_analysis/hits_alg.py:199: ComplexWarning: Casting complex values to real discards the imaginary part\n",
      "  authorities=dict(zip(G.nodes(),map(float,a)))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Run 2 size 80 for 500\n",
      "computing\n",
      "Run 3 size 31 for 500\n",
      "computing\n",
      "Run 4 size 56 for 500\n",
      "computing\n",
      "Run 5 size 45 for 500\n",
      "computing\n",
      "Run 6 size 58 for 500\n",
      "computing\n",
      "Run 7 size 35 for 500\n",
      "computing\n",
      "Run 8 size 54 for 500\n",
      "computing\n",
      "Run 9 size 172 for 500\n",
      "Run 10 size 40 for 500\n",
      "computing\n",
      "Run 11 size 1007 for 500\n",
      "Run 12 size 131 for 500\n",
      "computing\n",
      "Run 13 size 210 for 500\n",
      "Run 14 size 60 for 500\n",
      "computing\n",
      "Run 15 size 40 for 500\n",
      "computing\n",
      "Run 16 size 36 for 500\n",
      "computing\n",
      "Run 17 size 40 for 500\n",
      "computing\n",
      "Run 18 size 46 for 500\n",
      "computing\n",
      "Run 19 size 165 for 500\n",
      "Run 20 size 74 for 500\n",
      "computing\n",
      "Run 21 size 36 for 500\n",
      "computing\n",
      "Run 22 size 53 for 500\n",
      "computing\n",
      "Run 23 size 287 for 500\n",
      "Run 24 size 43 for 500\n",
      "computing\n",
      "Run 25 size 33 for 500\n",
      "computing\n",
      "Run 26 size 74 for 500\n",
      "computing\n",
      "Run 27 size 163 for 500\n",
      "Run 28 size 116 for 500\n",
      "computing\n",
      "Run 29 size 57 for 500\n",
      "computing\n",
      "Run 30 size 39 for 500\n",
      "computing\n",
      "Run 31 size 35 for 500\n",
      "computing\n",
      "Run 32 size 146 for 500\n",
      "computing\n",
      "Run 33 size 46 for 500\n",
      "computing\n",
      "Run 34 size 53 for 500\n",
      "computing\n",
      "Run 35 size 77 for 500\n",
      "computing\n",
      "Run 36 size 115 for 500\n",
      "computing\n",
      "Run 37 size 49 for 500\n",
      "computing\n",
      "Run 38 size 39 for 500\n",
      "computing\n",
      "Run 39 size 34 for 500\n",
      "computing\n",
      "Run 40 size 70 for 500\n",
      "computing\n",
      "Run 41 size 65 for 500\n",
      "computing\n",
      "Run 42 size 60 for 500\n",
      "computing\n",
      "Run 43 size 85 for 500\n",
      "computing\n",
      "Run 44 size 88 for 500\n",
      "computing\n",
      "Run 45 size 31 for 500\n",
      "computing\n",
      "Run 46 size 33 for 500\n",
      "computing\n",
      "Run 47 size 65 for 500\n",
      "computing\n",
      "Run 48 size 37 for 500\n",
      "computing\n",
      "Run 49 size 34 for 500\n",
      "computing\n",
      "Run 50 size 31 for 500\n",
      "computing\n",
      "Run 51 size 329 for 500\n",
      "Run 52 size 46 for 500\n",
      "computing\n",
      "Run 53 size 171 for 500\n",
      "Run 54 size 33 for 500\n",
      "computing\n",
      "Run 55 size 41 for 500\n",
      "computing\n",
      "Run 56 size 61 for 500\n",
      "computing\n",
      "Run 57 size 60 for 500\n",
      "computing\n",
      "Run 58 size 147 for 500\n",
      "computing\n",
      "Run 59 size 52 for 500\n",
      "computing\n",
      "Run 60 size 37 for 500\n",
      "computing\n",
      "Run 61 size 59 for 500\n",
      "computing\n",
      "Run 62 size 121 for 500\n",
      "computing\n",
      "Run 63 size 75 for 500\n",
      "computing\n",
      "Run 64 size 47 for 500\n",
      "computing\n",
      "Run 65 size 32 for 500\n",
      "computing\n",
      "Run 66 size 42 for 500\n",
      "computing\n",
      "Run 67 size 37 for 500\n",
      "computing\n",
      "Run 68 size 45 for 500\n",
      "computing\n",
      "Run 69 size 31 for 500\n",
      "computing\n",
      "Run 70 size 81 for 500\n",
      "computing\n",
      "Run 71 size 43 for 500\n",
      "computing\n",
      "Run 72 size 40 for 500\n",
      "computing\n",
      "Run 73 size 40 for 500\n",
      "computing\n",
      "Run 74 size 49 for 500\n",
      "computing\n",
      "Run 75 size 59 for 500\n",
      "computing\n",
      "Run 76 size 33 for 500\n",
      "computing\n",
      "Run 77 size 48 for 500\n",
      "computing\n",
      "Run 78 size 34 for 500\n",
      "computing\n",
      "Run 79 size 32 for 500\n",
      "computing\n",
      "Run 80 size 33 for 500\n",
      "computing\n",
      "Run 81 size 38 for 500\n",
      "computing\n",
      "Run 82 size 211 for 500\n",
      "Run 83 size 44 for 500\n",
      "computing\n",
      "Run 84 size 42 for 500\n",
      "computing\n",
      "Run 85 size 57 for 500\n",
      "computing\n",
      "Run 86 size 31 for 500\n",
      "computing\n",
      "Run 87 size 36 for 500\n",
      "computing\n",
      "Run 88 size 33 for 500\n",
      "computing\n",
      "Run 89 size 31 for 500\n",
      "computing\n",
      "Run 90 size 85 for 500\n",
      "computing\n",
      "Run 91 size 59 for 500\n",
      "computing\n",
      "Run 92 size 33 for 500\n",
      "computing\n",
      "Run 93 size 31 for 500\n",
      "computing\n",
      "Run 94 size 137 for 500\n",
      "computing\n",
      "Run 95 size 86 for 500\n",
      "computing\n",
      "Run 96 size 39 for 500\n",
      "computing\n",
      "Run 97 size 38 for 500\n",
      "computing\n",
      "Run 98 size 32 for 500\n",
      "computing\n",
      "Run 99 size 113 for 500\n",
      "computing\n",
      "Run 100 size 43 for 500\n",
      "computing\n",
      "Run 101 size 54 for 500\n",
      "computing\n",
      "Run 102 size 91 for 500\n",
      "computing\n",
      "Run 103 size 41 for 500\n",
      "computing\n",
      "Run 104 size 36 for 500\n",
      "computing\n",
      "Run 105 size 48 for 500\n",
      "computing\n",
      "Run 106 size 45 for 500\n",
      "computing\n",
      "Run 107 size 32 for 500\n",
      "computing\n",
      "Run 108 size 31 for 500\n",
      "computing\n",
      "Run 109 size 34 for 500\n",
      "computing\n",
      "Run 110 size 210 for 500\n",
      "Run 111 size 49 for 500\n",
      "computing\n",
      "Run 112 size 46 for 500\n",
      "computing\n",
      "Run 113 size 36 for 500\n",
      "computing\n",
      "Run 114 size 42 for 500\n",
      "computing\n",
      "Run 115 size 31 for 500\n",
      "computing\n",
      "Run 116 size 31 for 500\n",
      "computing\n",
      "Run 117 size 39 for 500\n",
      "computing\n",
      "Run 118 size 36 for 500\n",
      "computing\n",
      "Run 119 size 74 for 500\n",
      "computing\n",
      "Run 120 size 42 for 500\n",
      "computing\n",
      "Run 121 size 66 for 500\n",
      "computing\n",
      "Run 122 size 34 for 500\n",
      "computing\n",
      "Run 123 size 34 for 500\n",
      "computing\n",
      "Run 124 size 86 for 500\n",
      "computing\n",
      "Run 125 size 46 for 500\n",
      "computing\n",
      "Run 126 size 31 for 500\n",
      "computing\n",
      "Run 127 size 78 for 500\n",
      "computing\n",
      "Run 128 size 176 for 500\n",
      "Run 129 size 43 for 500\n",
      "computing\n",
      "Run 130 size 57 for 500\n",
      "computing\n",
      "Run 131 size 31 for 500\n",
      "computing\n",
      "Run 132 size 46 for 500\n",
      "computing\n",
      "Run 133 size 84 for 500\n",
      "computing\n",
      "Run 134 size 154 for 500\n",
      "Run 135 size 34 for 500\n",
      "computing\n",
      "Run 136 size 53 for 500\n",
      "computing\n",
      "Run 137 size 88 for 500\n",
      "computing\n",
      "Run 138 size 42 for 500\n",
      "computing\n",
      "Run 139 size 34 for 500\n",
      "computing\n",
      "Run 140 size 45 for 500\n",
      "computing\n",
      "Run 141 size 81 for 500\n",
      "computing\n",
      "Run 142 size 34 for 500\n",
      "computing\n",
      "Run 143 size 37 for 500\n",
      "computing\n",
      "Run 144 size 59 for 500\n",
      "computing\n",
      "Run 145 size 33 for 500\n",
      "computing\n",
      "Run 146 size 55 for 500\n",
      "computing\n",
      "Run 147 size 49 for 500\n",
      "computing\n",
      "Run 148 size 32 for 500\n",
      "computing\n",
      "Run 149 size 33 for 500\n",
      "computing\n",
      "Run 150 size 101 for 500\n",
      "computing\n",
      "Run 151 size 63 for 500\n",
      "computing\n",
      "Run 152 size 37 for 500\n",
      "computing\n",
      "Run 153 size 59 for 500\n",
      "computing\n",
      "Run 154 size 64 for 500\n",
      "computing\n",
      "Run 155 size 40 for 500\n",
      "computing\n",
      "Run 156 size 50 for 500\n",
      "computing\n",
      "Run 157 size 62 for 500\n",
      "computing\n",
      "Run 158 size 87 for 500\n",
      "computing\n",
      "Run 159 size 41 for 500\n",
      "computing\n",
      "Run 160 size 38 for 500\n",
      "computing\n",
      "Run 161 size 89 for 500\n",
      "computing\n",
      "Run 162 size 47 for 500\n",
      "computing\n",
      "Run 163 size 130 for 500\n",
      "computing\n",
      "Run 164 size 265 for 500\n",
      "Run 165 size 187 for 500\n",
      "Run 166 size 156 for 500\n",
      "Run 167 size 32 for 500\n",
      "computing\n",
      "Run 168 size 32 for 500\n",
      "computing\n",
      "Run 169 size 144 for 500\n",
      "computing\n",
      "Run 170 size 35 for 500\n",
      "computing\n",
      "Run 171 size 766 for 500\n",
      "Run 172 size 41 for 500\n",
      "computing\n",
      "Run 173 size 83 for 500\n",
      "computing\n",
      "Run 174 size 76 for 500\n",
      "computing\n",
      "Run 175 size 67 for 500\n",
      "computing\n",
      "Run 176 size 48 for 500\n",
      "computing\n",
      "Run 177 size 36 for 500\n",
      "computing\n",
      "Run 178 size 100 for 500\n",
      "computing\n",
      "Run 179 size 35 for 500\n",
      "computing\n",
      "Run 180 size 47 for 500\n",
      "computing\n",
      "Run 181 size 36 for 500\n",
      "computing\n",
      "Run 182 size 40 for 500\n",
      "computing\n",
      "Run 183 size 65 for 500\n",
      "computing\n",
      "Run 184 size 175 for 500\n",
      "Run 185 size 66 for 500\n",
      "computing\n",
      "Run 186 size 31 for 500\n",
      "computing\n",
      "Run 187 size 33 for 500\n",
      "computing\n",
      "Run 188 size 173 for 500\n",
      "Run 189 size 38 for 500\n",
      "computing\n",
      "Run 190 size 41 for 500\n",
      "computing\n",
      "Run 191 size 80 for 500\n",
      "computing\n",
      "Run 192 size 109 for 500\n",
      "computing\n",
      "Run 193 size 46 for 500\n",
      "computing\n",
      "Run 194 size 477 for 500\n",
      "Run 195 size 49 for 500\n",
      "computing\n",
      "Run 196 size 133 for 500\n",
      "computing\n",
      "Run 197 size 42 for 500\n",
      "computing\n",
      "Run 198 size 71 for 500\n",
      "computing\n",
      "Run 199 size 52 for 500\n",
      "computing\n",
      "Run 200 size 106 for 500\n",
      "computing\n",
      "Run 201 size 68 for 500\n",
      "computing\n",
      "Run 202 size 68 for 500\n",
      "computing\n",
      "Run 203 size 86 for 500\n",
      "computing\n",
      "Run 204 size 71 for 500\n",
      "computing\n",
      "Run 205 size 32 for 500\n",
      "computing\n",
      "Run 206 size 57 for 500\n",
      "computing\n",
      "Run 207 size 63 for 500\n",
      "computing\n",
      "Run 208 size 38 for 500\n",
      "computing\n",
      "Run 209 size 58 for 500\n",
      "computing\n",
      "Run 210 size 40 for 500\n",
      "computing\n",
      "Run 211 size 36 for 500\n",
      "computing\n",
      "Run 212 size 41 for 500\n",
      "computing\n",
      "Run 213 size 101 for 500\n",
      "computing\n",
      "Run 214 size 44 for 500\n",
      "computing\n",
      "Run 215 size 48 for 500\n",
      "computing\n",
      "Run 216 size 33 for 500\n",
      "computing\n",
      "Run 217 size 53 for 500\n",
      "computing\n",
      "Run 218 size 50 for 500\n",
      "computing\n",
      "Run 219 size 38 for 500\n",
      "computing\n",
      "Run 220 size 40 for 500\n",
      "computing\n",
      "Run 221 size 39 for 500\n",
      "computing\n",
      "Run 222 size 43 for 500\n",
      "computing\n",
      "Run 223 size 41 for 500\n",
      "computing\n",
      "Run 224 size 61 for 500\n",
      "computing\n",
      "Run 225 size 95 for 500\n",
      "computing\n",
      "Run 226 size 66 for 500\n",
      "computing\n",
      "Run 227 size 37 for 500\n",
      "computing\n",
      "Run 228 size 35 for 500\n",
      "computing\n",
      "Run 229 size 42 for 500\n",
      "computing\n",
      "Run 230 size 45 for 500\n",
      "computing\n",
      "Run 231 size 42 for 500\n",
      "computing\n",
      "Run 232 size 42 for 500\n",
      "computing\n",
      "Run 233 size 72 for 500\n",
      "computing\n",
      "Run 234 size 55 for 500\n",
      "computing\n",
      "Run 235 size 33 for 500\n",
      "computing\n",
      "Run 236 size 37 for 500\n",
      "computing\n",
      "Run 237 size 51 for 500\n",
      "computing\n",
      "Run 238 size 33 for 500\n",
      "computing\n",
      "Run 239 size 93 for 500\n",
      "computing\n",
      "Run 240 size 33 for 500\n",
      "computing\n",
      "Run 241 size 67 for 500\n",
      "computing\n",
      "Run 242 size 258 for 500\n",
      "Run 243 size 55 for 500\n",
      "computing\n",
      "Run 244 size 73 for 500\n",
      "computing\n",
      "Run 245 size 437 for 500\n",
      "Run 246 size 31 for 500\n",
      "computing\n",
      "Run 247 size 41 for 500\n",
      "computing\n",
      "Run 248 size 35 for 500\n",
      "computing\n",
      "Run 249 size 165 for 500\n",
      "Run 250 size 304 for 500\n",
      "Run 251 size 35 for 500\n",
      "computing\n",
      "Run 252 size 71 for 500\n",
      "computing\n",
      "Run 253 size 61 for 500\n",
      "computing\n",
      "Run 254 size 36 for 500\n",
      "computing\n",
      "Run 255 size 227 for 500\n",
      "Run 256 size 496 for 500\n",
      "Run 257 size 52 for 500\n",
      "computing\n",
      "Run 258 size 43 for 500\n",
      "computing\n",
      "Run 259 size 32 for 500\n",
      "computing\n",
      "Run 260 size 45 for 500\n",
      "computing\n",
      "Run 261 size 67 for 500\n",
      "computing\n",
      "Run 262 size 155 for 500\n",
      "Run 263 size 78 for 500\n",
      "computing\n",
      "Run 264 size 167 for 500\n",
      "Run 265 size 168 for 500\n",
      "Run 266 size 37 for 500\n",
      "computing\n",
      "Run 267 size 36 for 500\n",
      "computing\n",
      "Run 268 size 107 for 500\n",
      "computing\n",
      "Run 269 size 98 for 500\n",
      "computing\n",
      "Run 270 size 35 for 500\n",
      "computing\n",
      "Run 271 size 41 for 500\n",
      "computing\n",
      "Run 272 size 35 for 500\n",
      "computing\n",
      "Run 273 size 67 for 500\n",
      "computing\n",
      "Run 274 size 71 for 500\n",
      "computing\n",
      "Run 275 size 35 for 500\n",
      "computing\n",
      "Run 276 size 72 for 500\n",
      "computing\n",
      "Run 277 size 36 for 500\n",
      "computing\n",
      "Run 278 size 41 for 500\n",
      "computing\n",
      "Run 279 size 60 for 500\n",
      "computing\n",
      "Run 280 size 31 for 500\n",
      "computing\n",
      "Run 281 size 67 for 500\n",
      "computing\n",
      "Run 282 size 38 for 500\n",
      "computing\n",
      "Run 283 size 37 for 500\n",
      "computing\n",
      "Run 284 size 37 for 500\n",
      "computing\n",
      "Run 285 size 51 for 500\n",
      "computing\n",
      "Run 286 size 33 for 500\n",
      "computing\n",
      "Run 287 size 45 for 500\n",
      "computing\n",
      "Run 288 size 41 for 500\n",
      "computing\n",
      "Run 289 size 56 for 500\n",
      "computing\n",
      "Run 290 size 87 for 500\n",
      "computing\n",
      "Run 291 size 40 for 500\n",
      "computing\n",
      "Run 292 size 32 for 500\n",
      "computing\n",
      "Run 293 size 84 for 500\n",
      "computing\n",
      "Run 294 size 113 for 500\n",
      "computing\n",
      "Run 295 size 68 for 500\n",
      "computing\n",
      "Run 296 size 45 for 500\n",
      "computing\n",
      "Run 297 size 35 for 500\n",
      "computing\n",
      "Run 298 size 206 for 500\n",
      "Run 299 size 292 for 500\n",
      "Run 300 size 74 for 500\n",
      "computing\n",
      "Run 301 size 43 for 500\n",
      "computing\n",
      "Run 302 size 42 for 500\n",
      "computing\n",
      "Run 303 size 44 for 500\n",
      "computing\n",
      "Run 304 size 44 for 500\n",
      "computing\n",
      "Run 305 size 33 for 500\n",
      "computing\n",
      "Run 306 size 31 for 500\n",
      "computing\n",
      "Run 307 size 52 for 500\n",
      "computing\n",
      "Run 308 size 41 for 500\n",
      "computing\n",
      "Run 309 size 39 for 500\n",
      "computing\n",
      "Run 310 size 31 for 500\n",
      "computing\n",
      "Run 311 size 46 for 500\n",
      "computing\n",
      "Run 312 size 33 for 500\n",
      "computing\n",
      "Run 313 size 122 for 500\n",
      "computing\n",
      "Run 314 size 123 for 500\n",
      "computing\n",
      "Run 315 size 61 for 500\n",
      "computing\n",
      "Run 316 size 55 for 500\n",
      "computing\n",
      "Run 317 size 71 for 500\n",
      "computing\n",
      "Run 318 size 31 for 500\n",
      "computing\n",
      "Run 319 size 57 for 500\n",
      "computing\n",
      "Run 320 size 76 for 500\n",
      "computing\n",
      "Run 321 size 40 for 500\n",
      "computing\n",
      "Run 322 size 37 for 500\n",
      "computing\n",
      "Run 323 size 36 for 500\n",
      "computing\n",
      "Run 324 size 122 for 500\n",
      "computing\n",
      "Run 325 size 78 for 500\n",
      "computing\n",
      "Run 326 size 33 for 500\n",
      "computing\n",
      "Run 327 size 34 for 500\n",
      "computing\n",
      "Run 328 size 34 for 500\n",
      "computing\n",
      "Run 329 size 50 for 500\n",
      "computing\n",
      "Run 330 size 40 for 500\n",
      "computing\n",
      "Run 331 size 216 for 500\n",
      "Run 332 size 55 for 500\n",
      "computing\n",
      "Run 333 size 67 for 500\n",
      "computing\n",
      "Run 334 size 61 for 500\n",
      "computing\n",
      "Run 335 size 42 for 500\n",
      "computing\n",
      "Run 336 size 36 for 500\n",
      "computing\n",
      "Run 337 size 54 for 500\n",
      "computing\n",
      "Run 338 size 32 for 500\n",
      "computing\n",
      "Run 339 size 31 for 500\n",
      "computing\n",
      "Run 340 size 39 for 500\n",
      "computing\n",
      "Run 341 size 61 for 500\n",
      "computing\n",
      "Run 342 size 388 for 500\n",
      "Run 343 size 134 for 500\n",
      "computing\n",
      "Run 344 size 65 for 500\n",
      "computing\n",
      "Run 345 size 34 for 500\n",
      "computing\n",
      "Run 346 size 163 for 500\n",
      "Run 347 size 37 for 500\n",
      "computing\n",
      "Run 348 size 34 for 500\n",
      "computing\n",
      "Run 349 size 40 for 500\n",
      "computing\n",
      "Run 350 size 62 for 500\n",
      "computing\n",
      "Run 351 size 91 for 500\n",
      "computing\n",
      "Run 352 size 106 for 500\n",
      "computing\n",
      "Run 353 size 110 for 500\n",
      "computing\n",
      "Run 354 size 255 for 500\n",
      "Run 355 size 45 for 500\n",
      "computing\n",
      "Run 356 size 401 for 500\n",
      "Run 357 size 50 for 500\n",
      "computing\n",
      "Run 358 size 32 for 500\n",
      "computing\n",
      "Run 359 size 51 for 500\n",
      "computing\n",
      "Run 360 size 31 for 500\n",
      "computing\n",
      "Run 361 size 37 for 500\n",
      "computing\n",
      "Run 362 size 33 for 500\n",
      "computing\n",
      "Run 363 size 55 for 500\n",
      "computing\n",
      "Run 364 size 71 for 500\n",
      "computing\n",
      "Run 365 size 36 for 500\n",
      "computing\n",
      "Run 366 size 31 for 500\n",
      "computing\n",
      "Run 367 size 45 for 500\n",
      "computing\n",
      "Run 368 size 40 for 500\n",
      "computing\n",
      "Run 369 size 42 for 500\n",
      "computing\n",
      "Run 370 size 34 for 500\n",
      "computing\n",
      "Run 371 size 111 for 500\n",
      "computing\n",
      "Run 372 size 36 for 500\n",
      "computing\n",
      "Run 373 size 34 for 500\n",
      "computing\n",
      "Run 374 size 49 for 500\n",
      "computing\n",
      "Run 375 size 41 for 500\n",
      "computing\n",
      "Run 376 size 33 for 500\n",
      "computing\n",
      "Run 377 size 37 for 500\n",
      "computing\n",
      "Run 378 size 50 for 500\n",
      "computing\n",
      "Run 379 size 64 for 500\n",
      "computing\n",
      "Run 380 size 31 for 500\n",
      "computing\n",
      "Run 381 size 70 for 500\n",
      "computing\n",
      "Run 382 size 38 for 500\n",
      "computing\n",
      "Run 383 size 38 for 500\n",
      "computing\n",
      "Run 384 size 35 for 500\n",
      "computing\n",
      "Run 385 size 50 for 500\n",
      "computing\n",
      "Run 386 size 48 for 500\n",
      "computing\n",
      "Run 387 size 32 for 500\n",
      "computing\n",
      "Run 388 size 39 for 500\n",
      "computing\n",
      "Run 389 size 34 for 500\n",
      "computing\n",
      "Run 390 size 31 for 500\n",
      "computing\n",
      "Run 391 size 45 for 500\n",
      "computing\n",
      "Run 392 size 53 for 500\n",
      "computing\n",
      "Run 393 size 195 for 500\n",
      "Run 394 size 32 for 500\n",
      "computing\n",
      "Run 395 size 104 for 500\n",
      "computing\n",
      "Run 396 size 35 for 500\n",
      "computing\n",
      "Run 397 size 43 for 500\n",
      "computing\n",
      "Run 398 size 51 for 500\n",
      "computing\n",
      "Run 399 size 31 for 500\n",
      "computing\n",
      "Run 400 size 49 for 500\n",
      "computing\n",
      "Run 401 size 50 for 500\n",
      "computing\n",
      "Run 402 size 48 for 500\n",
      "computing\n",
      "Run 403 size 41 for 500\n",
      "computing\n",
      "Run 404 size 58 for 500\n",
      "computing\n",
      "Run 405 size 33 for 500\n",
      "computing\n",
      "Run 406 size 37 for 500\n",
      "computing\n",
      "Run 407 size 38 for 500\n",
      "computing\n",
      "Run 408 size 47 for 500\n",
      "computing\n",
      "Run 409 size 40 for 500\n",
      "computing\n",
      "Run 410 size 61 for 500\n",
      "computing\n",
      "Run 411 size 33 for 500\n",
      "computing\n",
      "Run 412 size 35 for 500\n",
      "computing\n",
      "Run 413 size 106 for 500\n",
      "computing\n",
      "Run 414 size 31 for 500\n",
      "computing\n",
      "Run 415 size 32 for 500\n",
      "computing\n",
      "Run 416 size 57 for 500\n",
      "computing\n",
      "Run 417 size 55 for 500\n",
      "computing\n",
      "Run 418 size 33 for 500\n",
      "computing\n",
      "Run 419 size 51 for 500\n",
      "computing\n",
      "Run 420 size 44 for 500\n",
      "computing\n",
      "Run 421 size 32 for 500\n",
      "computing\n",
      "Run 422 size 31 for 500\n",
      "computing\n",
      "Run 423 size 36 for 500\n",
      "computing\n",
      "Run 424 size 78 for 500\n",
      "computing\n",
      "Run 425 size 38 for 500\n",
      "computing\n",
      "Run 426 size 38 for 500\n",
      "computing\n",
      "Run 427 size 72 for 500\n",
      "computing\n",
      "Run 428 size 147 for 500\n",
      "computing\n",
      "Run 429 size 57 for 500\n",
      "computing\n",
      "Run 430 size 74 for 500\n",
      "computing\n",
      "Run 431 size 108 for 500\n",
      "computing\n",
      "Run 432 size 39 for 500\n",
      "computing\n",
      "Run 433 size 49 for 500\n",
      "computing\n",
      "Run 434 size 60 for 500\n",
      "computing\n",
      "Run 435 size 59 for 500\n",
      "computing\n",
      "Run 436 size 40 for 500\n",
      "computing\n",
      "Run 437 size 62 for 500\n",
      "computing\n",
      "Run 438 size 189 for 500\n",
      "Run 439 size 41 for 500\n",
      "computing\n",
      "Run 440 size 36 for 500\n",
      "computing\n",
      "Run 441 size 103 for 500\n",
      "computing\n",
      "Run 442 size 127 for 500\n",
      "computing\n",
      "Run 443 size 33 for 500\n",
      "computing\n",
      "Run 444 size 38 for 500\n",
      "computing\n",
      "Run 445 size 42 for 500\n",
      "computing\n",
      "Run 446 size 40 for 500\n",
      "computing\n",
      "Run 447 size 38 for 500\n",
      "computing\n",
      "Run 448 size 61 for 500\n",
      "computing\n",
      "Run 449 size 44 for 500\n",
      "computing\n",
      "Run 450 size 48 for 500\n",
      "computing\n",
      "Run 451 size 54 for 500\n",
      "computing\n",
      "Run 452 size 47 for 500\n",
      "computing\n",
      "Run 453 size 34 for 500\n",
      "computing\n",
      "Run 454 size 45 for 500\n",
      "computing\n",
      "Run 455 size 47 for 500\n",
      "computing\n",
      "Run 456 size 34 for 500\n",
      "computing\n",
      "Run 457 size 32 for 500\n",
      "computing\n",
      "Run 458 size 40 for 500\n",
      "computing\n",
      "Run 459 size 45 for 500\n",
      "computing\n",
      "Run 460 size 42 for 500\n",
      "computing\n",
      "Run 461 size 43 for 500\n",
      "computing\n",
      "Run 462 size 31 for 500\n",
      "computing\n",
      "Run 463 size 32 for 500\n",
      "computing\n",
      "Run 464 size 60 for 500\n",
      "computing\n",
      "Run 465 size 132 for 500\n",
      "computing\n",
      "Run 466 size 32 for 500\n",
      "computing\n",
      "Run 467 size 201 for 500\n",
      "Run 468 size 47 for 500\n",
      "computing\n",
      "Run 469 size 33 for 500\n",
      "computing\n",
      "Run 470 size 183 for 500\n",
      "Run 471 size 45 for 500\n",
      "computing\n",
      "Run 472 size 82 for 500\n",
      "computing\n",
      "Run 473 size 79 for 500\n",
      "computing\n",
      "Run 474 size 117 for 500\n",
      "computing\n",
      "Run 475 size 51 for 500\n",
      "computing\n",
      "Run 476 size 55 for 500\n",
      "computing\n",
      "Run 477 size 32 for 500\n",
      "computing\n",
      "Run 478 size 36 for 500\n",
      "computing\n",
      "Run 479 size 50 for 500\n",
      "computing\n",
      "Run 480 size 32 for 500\n",
      "computing\n",
      "Run 481 size 50 for 500\n",
      "computing\n",
      "Run 482 size 34 for 500\n",
      "computing\n",
      "Run 483 size 41 for 500\n",
      "computing\n",
      "Run 484 size 57 for 500\n",
      "computing\n",
      "Run 485 size 42 for 500\n",
      "computing\n",
      "Run 486 size 43 for 500\n",
      "computing\n",
      "Run 487 size 33 for 500\n",
      "computing\n",
      "Run 488 size 34 for 500\n",
      "computing\n",
      "Run 489 size 44 for 500\n",
      "computing\n",
      "Run 490 size 59 for 500\n",
      "computing\n",
      "Run 491 size 35 for 500\n",
      "computing\n",
      "Run 492 size 32 for 500\n",
      "computing\n",
      "Run 493 size 37 for 500\n",
      "computing\n",
      "Run 494 size 37 for 500\n",
      "computing\n",
      "Run 495 size 36 for 500\n",
      "computing\n",
      "Run 496 size 38 for 500\n",
      "computing\n",
      "Run 497 size 32 for 500\n",
      "computing\n",
      "Run 498 size 48 for 500\n",
      "computing\n",
      "Run 499 size 85 for 500\n",
      "computing\n",
      "Run 500 size 104 for 500\n",
      "computing\n"
     ]
    }
   ],
   "source": [
    "min_comments=30\n",
    "min_votes=5\n",
    "\n",
    "grouped=dfProducts[dfProducts['tot'].astype(int)>min_votes].groupby('asin')\n",
    "total = len(grouped)\n",
    "run = 1\n",
    "for name, group in grouped:\n",
    "    \n",
    "    dffiltro = (dfProducts['asin']==name) & (dfProducts['tot'].astype(int)>min_votes)\n",
    "    comments_count = dfProducts[dffiltro ]['tot'].values\n",
    "    \n",
    "    print(\"Run %d size %d for %d\" % (run, len(comments_count), total))\n",
    "    \n",
    "    if ( (len(comments_count)>min_comments) and (len(comments_count)<150) ):\n",
    "        sentences=[]\n",
    "        \n",
    "        print \"computing\"\n",
    "        \n",
    "        for t in dfProducts[dffiltro].T.to_dict().values():\n",
    "            for s in nltk.sent_tokenize(t['reviewText']):\n",
    "                sentences.append(compute_tags(s))   \n",
    "                \n",
    "        sentences_graph = compute_sentences_graph(sentences)\n",
    "        nx_graph = nx.from_numpy_matrix(sentences_graph)\n",
    "        pg_sentences = nx.pagerank_numpy(nx_graph)\n",
    "        hits_sentences = nx.hits_numpy(nx_graph)\n",
    "\n",
    "        count=0\n",
    "        for t in dfProducts[dffiltro].T.to_dict().values():\n",
    "            for s in nltk.sent_tokenize(t['reviewText']):\n",
    "                dfProducts.loc[t.values()[3],'pageRank'] += float(pg_sentences[count])\n",
    "                dfProducts.loc[t.values()[3],'hits'] += float(hits_sentences[1][count]) #authority score\n",
    "                count += 1\n",
    "\n",
    "        \n",
    "        #break\n",
    "    run += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dfProducts.to_csv('data/book_sample.csv.gz', compression='gzip')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}