IntegrationTest_Mallet_Export.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from dariah_topics import preprocessing as pre\n",
    "from dariah_topics import visualization as visual\n",
    "from dariah_topics import evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "#### Liste mit Dateinamen erzeugen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO preprocessing: Creating document list from TXT files ...\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['corpus_txt/Lovecraft_AttheMountainofMadness.txt',\n",
       " 'corpus_txt/Howard_TheDevilinIron.txt',\n",
       " 'corpus_txt/Poe_ThePurloinedLetter.txt',\n",
       " 'corpus_txt/Lovecraft_TheShunnedHouse.txt',\n",
       " 'corpus_txt/Poe_TheMasqueoftheRedDeath.txt']"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path_txt = \"corpus_txt\"\n",
    "#path_txt = \"grenzbote_plain/*/\"\n",
    "#path_txt = \"wiki/\"\n",
    "\n",
    "doclist_txt = pre.create_document_list(path_txt)\n",
    "assert doclist_txt, \"No documents found\"\n",
    "doclist_txt[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "#####  Liste mit Dokumentenlabels erzeugen - (Funktion wird durch Thorsten's generischere Funktion ersetzt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO preprocessing: Creating document labels ...\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['Lovecraft_AttheMountainofMadness',\n",
       " 'Howard_TheDevilinIron',\n",
       " 'Poe_ThePurloinedLetter',\n",
       " 'Lovecraft_TheShunnedHouse',\n",
       " 'Poe_TheMasqueoftheRedDeath']"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doc_labels = list(pre.get_labels(doclist_txt))\n",
    "doc_labels[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "#### Corpus laden"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "corpus_txt = pre.read_from_txt(doclist_txt)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Segmentieren"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<generator object segment_fuzzy at 0x7f77c36095c8>"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "segments = pre.segment(corpus_txt, 1000)\n",
    "segments\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "#### Tokenisieren"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO preprocessing: Accessing TXT documents ...\n"
     ]
    }
   ],
   "source": [
    "doc_tokens = [list(pre.tokenize(txt)) for txt in list(corpus_txt)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "#### Create Dictionaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'afeared': 1,\n",
       " 'hones': 4198,\n",
       " 'men-folk': 2,\n",
       " 'shuts': 4,\n",
       " 'slaty-blue': 5,\n",
       " 'creep': 6,\n",
       " 'feat': 7,\n",
       " 'brumby': 10,\n",
       " 'muscled': 11,\n",
       " 'ax': 12,\n",
       " 'mention': 13,\n",
       " 'finger-tips': 14,\n",
       " 'sloppy': 11143,\n",
       " 'calf': 16,\n",
       " 'rascals': 17,\n",
       " 'formless': 11142,\n",
       " 'beast': 18,\n",
       " 'stone-still': 19,\n",
       " 'pyramids': 21,\n",
       " 'globe': 22,\n",
       " 'datum': 23,\n",
       " 'donna': 24,\n",
       " 'saddles': 25,\n",
       " 'indulged': 26,\n",
       " 'convolvuluses': 27,\n",
       " 'fitful': 28,\n",
       " 'unhurt': 29,\n",
       " 'ugh': 30,\n",
       " 'scandalous': 31,\n",
       " 'wented-past': 32,\n",
       " 'tinted': 33,\n",
       " 'sill': 34,\n",
       " 'represented': 18920,\n",
       " 'bartholomew': 7413,\n",
       " 'jewels': 35,\n",
       " 'stenches': 36,\n",
       " 'tarry': 37,\n",
       " 'dot': 38,\n",
       " 'half-luminous': 39,\n",
       " 'much-discussed': 40,\n",
       " 'fog-wreaths': 41,\n",
       " 'acquiring': 42,\n",
       " 'starving': 43,\n",
       " 'habitual': 44,\n",
       " 'uniformly': 45,\n",
       " 'ratlook': 48,\n",
       " 'abruptness': 47,\n",
       " 'employer': 49,\n",
       " 'glad': 50,\n",
       " 'ran': 51,\n",
       " 'satellites': 52,\n",
       " 'vapor': 53,\n",
       " 'half-strangling': 54,\n",
       " 'chemical': 56,\n",
       " 'wedding': 57,\n",
       " 'selfish': 58,\n",
       " 'misfortunes': 59,\n",
       " 'tending': 60,\n",
       " 'wah': 61,\n",
       " 'borders': 62,\n",
       " 'visage': 63,\n",
       " 'goaded': 64,\n",
       " 'chambers': 65,\n",
       " 'complex': 7420,\n",
       " 'hind-foot': 66,\n",
       " 'dial': 67,\n",
       " 'black': 68,\n",
       " 'spots': 69,\n",
       " 'proof': 70,\n",
       " 'guiacum': 71,\n",
       " 'transcribing': 72,\n",
       " 'red-gold': 18638,\n",
       " 'supplied': 15,\n",
       " 'impassioned': 74,\n",
       " 'duplicate': 75,\n",
       " 'dunkeln': 76,\n",
       " 'chimney': 77,\n",
       " 'convulsively': 78,\n",
       " 'meditation': 80,\n",
       " 'clearances': 81,\n",
       " 'preceptor': 9540,\n",
       " 'miners': 14279,\n",
       " 'yard': 82,\n",
       " 'fissure': 83,\n",
       " 'presumptuous': 84,\n",
       " 'involve': 85,\n",
       " 'arulala': 86,\n",
       " 'filfy': 87,\n",
       " 'fastened': 89,\n",
       " 'hydrophobia': 90,\n",
       " 'girths-loosed': 91,\n",
       " 'foot-fall': 92,\n",
       " 'joseph': 93,\n",
       " 'unloosed': 94,\n",
       " 'vague': 95,\n",
       " 'kite': 96,\n",
       " 'seize': 97,\n",
       " 'village-alive': 3715,\n",
       " 'scurry': 98,\n",
       " 'taxmen': 100,\n",
       " 'sealed': 101,\n",
       " 'fully': 14880,\n",
       " 'wriggle': 102,\n",
       " 'billion': 103,\n",
       " 'familiarized': 104,\n",
       " 'remain': 105,\n",
       " 'clipped': 106,\n",
       " \"wulfhere's\": 107,\n",
       " 'statu': 7426,\n",
       " 'holy': 20,\n",
       " 'heralded': 108,\n",
       " 'accommodate': 109,\n",
       " 'discomforted': 11306,\n",
       " 'emits': 110,\n",
       " 'guardian': 111,\n",
       " 'revolutions': 112,\n",
       " 'antecedents': 113,\n",
       " 'effusion': 115,\n",
       " 'champed': 116,\n",
       " 'favorable': 117,\n",
       " 'crossroads': 119,\n",
       " 'faubourg': 120,\n",
       " 'atwood': 121,\n",
       " 'grossen': 122,\n",
       " 'famille': 124,\n",
       " 'presently': 125,\n",
       " 'vast': 126,\n",
       " 'questioners': 127,\n",
       " 'calendar': 128,\n",
       " 'gravely': 129,\n",
       " 'spilt': 3725,\n",
       " 'slovenly': 130,\n",
       " 'corinthian': 131,\n",
       " 'accepted': 132,\n",
       " 'flora': 133,\n",
       " 'animate': 134,\n",
       " 'unfavorable': 135,\n",
       " 'biting': 136,\n",
       " 'marks': 137,\n",
       " 'assortment': 140,\n",
       " 'vixen': 141,\n",
       " \"argument's\": 142,\n",
       " 'dick': 143,\n",
       " \"shouldn't\": 144,\n",
       " 'intervals': 145,\n",
       " 'lusty': 146,\n",
       " 'cold-resistant': 147,\n",
       " 'rioting': 148,\n",
       " 'directness': 149,\n",
       " 'waistcoat': 150,\n",
       " 'opposed': 14889,\n",
       " 'twined': 151,\n",
       " 'motion': 153,\n",
       " 'monographs': 157,\n",
       " 'patagonia-way': 156,\n",
       " 'vertebrae': 158,\n",
       " 'fairy': 159,\n",
       " 'fetching': 160,\n",
       " 'solchen': 162,\n",
       " 'abou': 163,\n",
       " 'refusal': 164,\n",
       " 'ranchman': 18652,\n",
       " 'ambiguity': 165,\n",
       " 'sojourn': 3733,\n",
       " 'dis': 166,\n",
       " 'biting-out': 167,\n",
       " 'long-horned': 168,\n",
       " 'surge': 169,\n",
       " 'straight-walking': 170,\n",
       " 'harmony': 171,\n",
       " 'counterbalance': 172,\n",
       " 'romped': 173,\n",
       " 'telling': 174,\n",
       " 'friendless': 175,\n",
       " 'peacefully': 176,\n",
       " 'realized': 177,\n",
       " 'ale-house': 178,\n",
       " 'balustraded': 179,\n",
       " 'collar': 11169,\n",
       " 'clean-cut': 180,\n",
       " 'sings': 181,\n",
       " 'donations': 183,\n",
       " 'unhad': 184,\n",
       " 'jack-in': 185,\n",
       " 'panes': 186,\n",
       " 'top-coated': 187,\n",
       " 'smiths': 188,\n",
       " 'pelties': 11174,\n",
       " 'rosemary': 189,\n",
       " 'ravager-son': 18658,\n",
       " \"drebber's\": 190,\n",
       " 'fellow-atom': 3737,\n",
       " 'lengths': 191,\n",
       " 'bland': 192,\n",
       " 'latterly': 193,\n",
       " 'deception': 194,\n",
       " 'killer-whale': 195,\n",
       " 'rock': 7298,\n",
       " 'proclaimed': 196,\n",
       " 'coughed': 197,\n",
       " 'islands': 198,\n",
       " 'violin-playing': 199,\n",
       " 'picked': 200,\n",
       " 'radicalness': 201,\n",
       " \"she'll\": 203,\n",
       " 'ouch': 204,\n",
       " 'dancer': 208,\n",
       " 'lovies': 206,\n",
       " 'hotfoot': 209,\n",
       " 'sicked-up': 7445,\n",
       " 'offices': 210,\n",
       " 'yearling': 211,\n",
       " 'michaelmas': 212,\n",
       " 'slipped': 14896,\n",
       " 'fireside': 18663,\n",
       " 'outbreak': 213,\n",
       " 'liver': 214,\n",
       " 'savages': 215,\n",
       " 'all-of': 216,\n",
       " 'remarkable-looking': 217,\n",
       " 'wincing': 218,\n",
       " 'sleeped': 219,\n",
       " 'sadness': 220,\n",
       " 'fortunes': 7096,\n",
       " 'democracy': 221,\n",
       " 'bending': 222,\n",
       " 'goods': 223,\n",
       " 'glut': 224,\n",
       " 'foulmire': 228,\n",
       " \"i'll\": 227,\n",
       " 'insidious': 229,\n",
       " 'gaelic': 230,\n",
       " 'mutterings': 231,\n",
       " 'array': 232,\n",
       " 'mesh-mail': 233,\n",
       " 'residence': 234,\n",
       " 'ac': 235,\n",
       " 'made': 236,\n",
       " 'somewhere': 237,\n",
       " 'spade': 14899,\n",
       " 'prearranged': 238,\n",
       " 'rang': 239,\n",
       " 'triassic': 240,\n",
       " 'measuring': 241,\n",
       " 'dauntless': 242,\n",
       " 'plantation': 245,\n",
       " 'bluish': 244,\n",
       " 'shore-line': 18670,\n",
       " 'lunge': 246,\n",
       " 'gedney': 248,\n",
       " 'drooped': 249,\n",
       " 'standing': 251,\n",
       " 'kingdoms': 252,\n",
       " \"woodcutter's\": 253,\n",
       " 'glass-topped': 254,\n",
       " 'destruction': 255,\n",
       " 'soundlessly': 256,\n",
       " 'expired': 259,\n",
       " 'red-sashed': 260,\n",
       " 'theories': 261,\n",
       " 'appeasing': 46,\n",
       " 'resident': 262,\n",
       " 'mongrels': 263,\n",
       " 'showman': 264,\n",
       " 'increases': 14902,\n",
       " 'leader': 265,\n",
       " \"you're\": 266,\n",
       " 'die': 267,\n",
       " 'listens': 268,\n",
       " 'lucid': 269,\n",
       " 'draw': 271,\n",
       " 'fringe': 272,\n",
       " 'recapture': 273,\n",
       " 'complimented': 274,\n",
       " 'highclass': 275,\n",
       " 'amber-and': 276,\n",
       " 'darkened': 277,\n",
       " 'phenomena': 278,\n",
       " 'solving': 279,\n",
       " 'spectrally': 280,\n",
       " 'rendered': 281,\n",
       " \"friend's\": 282,\n",
       " 'owls': 3754,\n",
       " 'hen-killer': 284,\n",
       " 'self-cognizance': 286,\n",
       " 'deride': 18675,\n",
       " 'eggy': 287,\n",
       " 'cul-de': 3756,\n",
       " 'imposed': 288,\n",
       " 'tar': 290,\n",
       " 'despotism': 292,\n",
       " 'depicted': 293,\n",
       " 'unaccountability': 18676,\n",
       " 'skilful': 295,\n",
       " 'turn': 296,\n",
       " 'smart': 297,\n",
       " 'sorrowfully': 13396,\n",
       " 'tendo': 298,\n",
       " 'operated': 299,\n",
       " 'finish': 300,\n",
       " 'drillings': 301,\n",
       " 'climates': 303,\n",
       " 'ghastly': 304,\n",
       " 'labors': 305,\n",
       " 'four-wheeler': 307,\n",
       " 'leer': 308,\n",
       " \"papa's\": 310,\n",
       " 'opportunities': 311,\n",
       " 'pterodactyl': 312,\n",
       " 'laughing': 313,\n",
       " 'population': 18679,\n",
       " 'giant': 55,\n",
       " 'fore-and': 314,\n",
       " 'oranges': 18682,\n",
       " \"fellow's\": 315,\n",
       " 'wired': 316,\n",
       " 'kadath': 317,\n",
       " 'damper': 318,\n",
       " 'neglect': 320,\n",
       " 'mired': 322,\n",
       " 'cuttings': 323,\n",
       " 'damp': 325,\n",
       " 'stool': 326,\n",
       " 'point': 327,\n",
       " 'betrothal': 328,\n",
       " 'placed': 329,\n",
       " 'ungraciously': 11194,\n",
       " 'waddled': 330,\n",
       " 'hand': 331,\n",
       " 'kamya': 333,\n",
       " \"rope's\": 335,\n",
       " 'mealy-mouthed': 334,\n",
       " 'conjure': 336,\n",
       " 'imaginable': 337,\n",
       " 'floodtime': 339,\n",
       " 'dant': 340,\n",
       " 'ulysses': 341,\n",
       " 'grizzled': 342,\n",
       " 'correct': 343,\n",
       " 'bet': 344,\n",
       " 'disciples': 345,\n",
       " 'amazed': 347,\n",
       " 'shes': 348,\n",
       " 'vehemence': 349,\n",
       " 'ceasing': 350,\n",
       " 'corps': 351,\n",
       " 'dost': 352,\n",
       " 'afterward': 353,\n",
       " 'breezes': 354,\n",
       " 'seal-skin': 355,\n",
       " 'villainy': 356,\n",
       " 'surgeon': 357,\n",
       " 'overlook': 358,\n",
       " 'metaphysical': 359,\n",
       " 'cöalitions': 360,\n",
       " 'unlighted': 361,\n",
       " 'montresor': 362,\n",
       " 'laborious': 363,\n",
       " 'life-boat': 365,\n",
       " 'habitat': 366,\n",
       " 'slush': 367,\n",
       " 'vista': 368,\n",
       " 'piquancy': 11200,\n",
       " 'township': 369,\n",
       " 'trincomalee': 370,\n",
       " 'coffin-lid': 371,\n",
       " 'needless': 372,\n",
       " 'dash-fine': 373,\n",
       " 'parties': 374,\n",
       " 'rikk-tck': 375,\n",
       " 'symmetry': 6420,\n",
       " 'troubles': 377,\n",
       " 'view': 378,\n",
       " 'conceded': 379,\n",
       " 'cross-examined': 380,\n",
       " 'rebuttoned': 15782,\n",
       " 'believed': 381,\n",
       " 'extinct': 382,\n",
       " 'crowning': 383,\n",
       " 'acquired': 384,\n",
       " 'celibate': 14915,\n",
       " 'specification': 385,\n",
       " 'jessamine': 386,\n",
       " 'antiquated': 387,\n",
       " \"cmb's\": 388,\n",
       " 'historical': 389,\n",
       " 'sleepest': 390,\n",
       " 'face': 11206,\n",
       " 'maddest': 392,\n",
       " 'accessory': 393,\n",
       " 'twig': 394,\n",
       " 'portray': 395,\n",
       " 'nearness': 397,\n",
       " 'successful': 399,\n",
       " 'jumps': 400,\n",
       " 'waste-paper': 402,\n",
       " 'ware-chicken': 403,\n",
       " 'romeo': 404,\n",
       " 'vices': 405,\n",
       " 'dogmaticians': 406,\n",
       " 'vast-enough': 407,\n",
       " 'warranted': 408,\n",
       " 'brute': 409,\n",
       " 'unreached': 410,\n",
       " 'gash': 412,\n",
       " 'acme': 413,\n",
       " 'sleep-in': 414,\n",
       " 'sprig': 11211,\n",
       " 'travels': 416,\n",
       " 'sheering': 417,\n",
       " 'freeboard': 418,\n",
       " 'truncated': 419,\n",
       " 'pump': 420,\n",
       " 'transcriber': 14921,\n",
       " 'sloop': 421,\n",
       " 'substantial': 422,\n",
       " \"hutchinson's\": 423,\n",
       " 'troop': 11857,\n",
       " 'rawal': 424,\n",
       " 'undo': 425,\n",
       " 'deserting': 426,\n",
       " 'critically': 427,\n",
       " 'streams': 428,\n",
       " 'mutual': 429,\n",
       " 'splits': 430,\n",
       " 'reporting': 431,\n",
       " 'strait-jacket': 432,\n",
       " 'canyons': 433,\n",
       " 'terrace': 434,\n",
       " 'recuperation': 435,\n",
       " 'flatter': 437,\n",
       " 'stock-undoubtedly': 438,\n",
       " 'dismal': 439,\n",
       " 'reeving': 440,\n",
       " 'particles': 441,\n",
       " 'refined': 442,\n",
       " 'entrance': 444,\n",
       " 'german-speaking': 445,\n",
       " 'facts': 446,\n",
       " 'towered': 447,\n",
       " 'malignant': 448,\n",
       " 'packing': 449,\n",
       " 'conceived': 450,\n",
       " 'sally': 452,\n",
       " 'unhealthy': 73,\n",
       " 'hangs': 453,\n",
       " 'strenuous': 455,\n",
       " 'train': 457,\n",
       " 'downward': 458,\n",
       " 'omniscience': 459,\n",
       " 'groping': 460,\n",
       " 'enabled': 8890,\n",
       " 'corridor': 461,\n",
       " 'chair': 463,\n",
       " 'dissent': 464,\n",
       " 'frenzy': 465,\n",
       " 'hinted': 466,\n",
       " 'assure': 467,\n",
       " 'unfrequently': 468,\n",
       " 'translucent': 469,\n",
       " 'smoke-rocket': 470,\n",
       " 'four-wheeled': 471,\n",
       " 'manservant': 472,\n",
       " 'thumb-mark': 474,\n",
       " 'impassable': 475,\n",
       " 'processes': 476,\n",
       " \"day's\": 477,\n",
       " 'brooch': 478,\n",
       " 'co': 479,\n",
       " 'lover': 481,\n",
       " 'pronounce': 79,\n",
       " 'ordinance': 14933,\n",
       " 'magnificent': 482,\n",
       " 'structures': 485,\n",
       " 'penang': 486,\n",
       " 'analogous': 487,\n",
       " 'coward': 14936,\n",
       " 'prevails': 488,\n",
       " 'abyssward': 18710,\n",
       " 'decadence': 490,\n",
       " 'occurred': 493,\n",
       " 'ample': 494,\n",
       " 'semaphoring': 495,\n",
       " 'alibi': 496,\n",
       " 'graves': 497,\n",
       " 'unaccounted': 499,\n",
       " 'lamentable': 502,\n",
       " 'cigar': 501,\n",
       " 'dairy': 503,\n",
       " \"arm's\": 3257,\n",
       " 'look-see': 504,\n",
       " 'likes': 505,\n",
       " 'wooden-legged': 506,\n",
       " 'chuchundra': 507,\n",
       " 'wharfingers': 508,\n",
       " 'peaky': 509,\n",
       " 'general': 510,\n",
       " 'extensive': 511,\n",
       " 'boy': 512,\n",
       " 'thinness': 513,\n",
       " 'normality': 514,\n",
       " 'folklorist': 515,\n",
       " 'beyondness': 516,\n",
       " 'hamjuice': 517,\n",
       " 'water-pipe': 11226,\n",
       " 'dissolving': 14943,\n",
       " 'third': 518,\n",
       " 'reports': 519,\n",
       " 'breakfast': 520,\n",
       " 'blinked': 521,\n",
       " 'argue': 522,\n",
       " 'raj': 523,\n",
       " 'man-pack': 527,\n",
       " 'lying': 529,\n",
       " 'contributed': 530,\n",
       " 'austere': 531,\n",
       " 'hollows': 3795,\n",
       " 'omit': 3791,\n",
       " 'gas': 533,\n",
       " 'pedlar': 534,\n",
       " 'yacht': 535,\n",
       " 'venerable': 536,\n",
       " 'gallop': 538,\n",
       " 'fox': 3796,\n",
       " 'mutilated': 539,\n",
       " 'hereby': 540,\n",
       " 'invaders': 541,\n",
       " 'hatched': 542,\n",
       " 'pseudofoot': 543,\n",
       " 'over-exposure': 544,\n",
       " 'plots': 545,\n",
       " 'notified': 546,\n",
       " 'accounted': 3131,\n",
       " 'jim': 548,\n",
       " 'elephant-folk': 549,\n",
       " 'mahomet': 550,\n",
       " 'peasants': 551,\n",
       " 'finesse': 552,\n",
       " 'leadership': 553,\n",
       " 'tree-tops': 554,\n",
       " 'load': 555,\n",
       " 'heavily': 556,\n",
       " 'pierril': 557,\n",
       " 'labouring': 1440,\n",
       " 'suspicious': 558,\n",
       " 'smallness': 559,\n",
       " 'precautions': 560,\n",
       " 'stamp': 7496,\n",
       " 'gravitating': 561,\n",
       " 'engine-room': 562,\n",
       " 'follows': 563,\n",
       " 'tiller': 565,\n",
       " 'day-and': 567,\n",
       " 'stacked': 568,\n",
       " 'clothe': 3803,\n",
       " 'well-set': 12273,\n",
       " 'poncho': 99,\n",
       " 'all-us': 570,\n",
       " 'generally': 571,\n",
       " 'cities': 572,\n",
       " 'diminishes': 574,\n",
       " 'fear': 18632,\n",
       " 'jowels': 575,\n",
       " 'dies': 576,\n",
       " 'either': 577,\n",
       " 'hardness': 578,\n",
       " 'avenue': 579,\n",
       " 'velvet-lined': 581,\n",
       " 'late-built': 583,\n",
       " \"fritz's\": 586,\n",
       " 'certainty': 588,\n",
       " 'impede': 587,\n",
       " 'outpouring': 10219,\n",
       " 'commit': 589,\n",
       " 'untied': 590,\n",
       " 'rushes': 591,\n",
       " \"we've\": 592,\n",
       " 'athelney': 594,\n",
       " 'templed': 595,\n",
       " 'challenges': 596,\n",
       " 'busily': 598,\n",
       " 'typewriter': 601,\n",
       " 'unkennel': 599,\n",
       " 'bohème': 600,\n",
       " 'warder': 602,\n",
       " 'married': 7501,\n",
       " 'jarred': 603,\n",
       " 'thoroughness': 604,\n",
       " 'attempting': 605,\n",
       " 'conceive': 606,\n",
       " 'waterproof': 607,\n",
       " 'openness': 608,\n",
       " 'bitterns': 611,\n",
       " 'honours': 612,\n",
       " 'wildest': 613,\n",
       " 'detestable': 614,\n",
       " 'unbosomed': 617,\n",
       " 'quarry': 616,\n",
       " 'gaze': 618,\n",
       " 'whom': 619,\n",
       " 'strolled': 620,\n",
       " \"moulton's\": 622,\n",
       " 'babies': 623,\n",
       " 'dwarfing': 624,\n",
       " 'equestrian': 626,\n",
       " 'recruit-mule': 629,\n",
       " 'knifing': 628,\n",
       " 'downs': 9303,\n",
       " 'coping': 630,\n",
       " 'determined': 631,\n",
       " 'engulfed': 7724,\n",
       " 'quills': 19589,\n",
       " 'ripen': 632,\n",
       " 'reiterated': 633,\n",
       " 'ought': 634,\n",
       " 'reproducing': 635,\n",
       " 'evidently': 636,\n",
       " 'but': 637,\n",
       " 'dream-land': 638,\n",
       " 'confirming': 639,\n",
       " 'colossally': 640,\n",
       " 'unbecomingly': 641,\n",
       " 'sallow': 11247,\n",
       " 'inn': 642,\n",
       " 'coarseness': 643,\n",
       " 'augment': 644,\n",
       " 'signed': 645,\n",
       " 'hatred': 646,\n",
       " 'corrientes': 3818,\n",
       " 'guts-of': 11029,\n",
       " 'amplified': 647,\n",
       " 'jingled': 648,\n",
       " 'pure': 649,\n",
       " 'dam-all': 650,\n",
       " 'recurrence': 651,\n",
       " 'partook': 652,\n",
       " 'jumping-on': 22204,\n",
       " 'produce': 653,\n",
       " 'louder': 654,\n",
       " 'plagues': 655,\n",
       " 'pertinaciously': 660,\n",
       " 'eg': 658,\n",
       " 'crawls': 114,\n",
       " 'rivals': 3820,\n",
       " 'approaching': 661,\n",
       " 'moidores': 18738,\n",
       " 'deep': 663,\n",
       " 'delude': 664,\n",
       " 'difficulties': 665,\n",
       " 'cöexist': 666,\n",
       " 'thus': 667,\n",
       " 'asked': 668,\n",
       " 'brunette': 670,\n",
       " 'braces': 671,\n",
       " 'extent': 672,\n",
       " 'traditional': 673,\n",
       " 'elucidate': 674,\n",
       " 'want': 11250,\n",
       " 'designs': 675,\n",
       " 'insidiously': 11252,\n",
       " 'outwards': 676,\n",
       " 'tribal': 677,\n",
       " 'believers': 7506,\n",
       " 'dusty-looking': 678,\n",
       " 'secreting': 679,\n",
       " 'sleep-hunting': 7860,\n",
       " 'wanting': 682,\n",
       " 'calmness': 683,\n",
       " 'dollies': 684,\n",
       " 'months': 685,\n",
       " 'grading': 686,\n",
       " 'far-off': 689,\n",
       " 'scribbled': 688,\n",
       " 'mineral': 691,\n",
       " 'coiled': 692,\n",
       " 'harsh': 11459,\n",
       " 'intuition': 693,\n",
       " 'imperfectly': 694,\n",
       " 'bearded': 695,\n",
       " 'forbade': 7511,\n",
       " 'wrote': 696,\n",
       " 'soles': 697,\n",
       " 'sex': 698,\n",
       " 'rend': 699,\n",
       " 'kepler': 701,\n",
       " 'larsen': 703,\n",
       " 'self-slaughter': 702,\n",
       " 'pistols': 18743,\n",
       " 'safest': 704,\n",
       " 'arranged': 705,\n",
       " 'undertook': 708,\n",
       " 'disconcert': 709,\n",
       " 'disconsolate': 710,\n",
       " 'trial': 711,\n",
       " 'galaxy': 712,\n",
       " 'dislodged': 7516,\n",
       " 'prowling': 713,\n",
       " 'silences': 714,\n",
       " 'survived': 715,\n",
       " 'hardship': 716,\n",
       " 'landslide': 717,\n",
       " 'barrelled': 718,\n",
       " 'heh': 11257,\n",
       " 'accursed': 6845,\n",
       " 'swings': 7517,\n",
       " 'listenin': 725,\n",
       " 'khorala': 724,\n",
       " 'swish': 721,\n",
       " 'burrowed': 722,\n",
       " 'heated': 723,\n",
       " 'kya-a': 726,\n",
       " 'bogie': 727,\n",
       " 'murderous': 118,\n",
       " 'tree-roads': 728,\n",
       " 'frankly': 729,\n",
       " 'heart-subduing': 730,\n",
       " 'fellow-mortal': 731,\n",
       " 'flotilla': 732,\n",
       " 'wildness': 1475,\n",
       " 'flaxen': 733,\n",
       " 'sailor': 734,\n",
       " 'tools': 737,\n",
       " 'mound-heaps': 738,\n",
       " 'sermon': 14968,\n",
       " 'avidly': 739,\n",
       " 'inaccessible': 740,\n",
       " 'snibbed': 741,\n",
       " 'stoopids': 742,\n",
       " 'joint': 743,\n",
       " 'gelatinous': 744,\n",
       " 'disperse': 747,\n",
       " 'distended': 746,\n",
       " 'asthmatic': 748,\n",
       " 'ubiquitous': 749,\n",
       " 'importunate': 750,\n",
       " 'bittern': 123,\n",
       " 'gagged': 751,\n",
       " 'bleaker': 7523,\n",
       " 'buttered': 752,\n",
       " 'challenge': 753,\n",
       " 'informant': 754,\n",
       " 'serving': 755,\n",
       " 'ramifications': 756,\n",
       " 'regularity': 11263,\n",
       " 'bemused': 757,\n",
       " 'mogul': 758,\n",
       " 'perceptibly': 759,\n",
       " 'plot': 760,\n",
       " 'fro': 761,\n",
       " 'vacillating': 762,\n",
       " 'puddling': 763,\n",
       " 'last-run': 765,\n",
       " 'skulk': 766,\n",
       " 'prettily': 769,\n",
       " 'speckled': 770,\n",
       " 'lullaby': 771,\n",
       " 'wilful': 772,\n",
       " 'resort': 773,\n",
       " 'replacing': 774,\n",
       " 'vines': 776,\n",
       " 'australia': 778,\n",
       " 'immensity': 779,\n",
       " 'scrutiny': 780,\n",
       " 'entering': 18756,\n",
       " 'fiendishness': 782,\n",
       " 'landing-parties': 783,\n",
       " 'dissolve': 784,\n",
       " 'third-rate': 786,\n",
       " 'gateway': 787,\n",
       " 'seventeenth': 788,\n",
       " 'ib': 789,\n",
       " 'schoolboy': 790,\n",
       " 'hen-ladies': 792,\n",
       " 'harp': 793,\n",
       " 'fuel-warming': 794,\n",
       " 'curve': 795,\n",
       " 'parrot': 20801,\n",
       " 'daised': 796,\n",
       " 'hell-hound': 797,\n",
       " 'sounding': 798,\n",
       " 'under-the': 799,\n",
       " 'prison-officials': 801,\n",
       " 'sophistry': 802,\n",
       " 'miscarried': 803,\n",
       " 'neatly': 804,\n",
       " 'complicates': 805,\n",
       " 'apologize': 806,\n",
       " 'yells': 807,\n",
       " 'brown-and': 808,\n",
       " 'ophir': 809,\n",
       " 'naive': 810,\n",
       " 'sombre': 811,\n",
       " 'draught': 11274,\n",
       " 'miracles': 816,\n",
       " 'ireland': 819,\n",
       " 'saddlery': 817,\n",
       " 'masts': 818,\n",
       " 'desisted': 821,\n",
       " 'leeds': 822,\n",
       " 'imported': 14982,\n",
       " 'ice-vapors': 823,\n",
       " 'dog-gent': 824,\n",
       " 'framed': 825,\n",
       " 'reptiles': 827,\n",
       " 'flame-eyed': 829,\n",
       " 'spurious': 830,\n",
       " 'illustrated': 831,\n",
       " 'window-pane': 832,\n",
       " 'unfair': 833,\n",
       " 'happy-dog': 834,\n",
       " 'nonsense': 139,\n",
       " 'breakfast-table': 835,\n",
       " 'heavenly': 836,\n",
       " 'livid': 837,\n",
       " 'length': 838,\n",
       " 'heedlessly': 839,\n",
       " 'pitiful': 11280,\n",
       " 'l’essence': 14985,\n",
       " 'disentangling': 840,\n",
       " 'ruddier': 841,\n",
       " 'olivia': 843,\n",
       " 'dry': 844,\n",
       " 'screws': 845,\n",
       " 'teachings': 846,\n",
       " 'womanly': 847,\n",
       " 'bungalow': 848,\n",
       " 'inferences': 850,\n",
       " 'course': 851,\n",
       " 'wooden': 853,\n",
       " 'abundant': 854,\n",
       " 'panted': 855,\n",
       " 'hen': 856,\n",
       " 'reconnoitering': 858,\n",
       " 'skulking': 859,\n",
       " 'teaspoonful': 7610,\n",
       " 'cywil': 860,\n",
       " 'patch': 861,\n",
       " 'night-night': 862,\n",
       " 'post': 11287,\n",
       " 'spinning-wheels': 863,\n",
       " 'dolphin': 865,\n",
       " 'luxuries': 11286,\n",
       " 'dewanee': 866,\n",
       " 'lestrade': 867,\n",
       " 'stables': 14994,\n",
       " 'enabling': 868,\n",
       " 'empress': 869,\n",
       " 'rake': 871,\n",
       " 'stunned': 10942,\n",
       " 'flare': 872,\n",
       " \"d'or\": 10157,\n",
       " 'disturb': 873,\n",
       " 'navigation': 874,\n",
       " 'shown': 3859,\n",
       " 'seafaring': 875,\n",
       " 'petulant': 876,\n",
       " 'foci': 878,\n",
       " 'forty-seven': 879,\n",
       " 'protracted': 881,\n",
       " 'binder': 882,\n",
       " 'bier': 883,\n",
       " 'field': 884,\n",
       " 'cracked': 885,\n",
       " 'hug': 886,\n",
       " 'sailing-orders': 888,\n",
       " 'regent': 889,\n",
       " 'thereafter': 890,\n",
       " 'overhanging': 10484,\n",
       " 'pounded': 891,\n",
       " 'sleepless': 7542,\n",
       " 'intelligently': 892,\n",
       " \"servant's\": 152,\n",
       " 'lunching': 893,\n",
       " 'fatigue': 894,\n",
       " 'scrambled': 895,\n",
       " 'equals': 896,\n",
       " 'art-product': 897,\n",
       " 'rubbery': 898,\n",
       " 'planes': 13613,\n",
       " 'spur': 899,\n",
       " 'projecting': 900,\n",
       " 'well-trained': 901,\n",
       " 'amended': 154,\n",
       " 'unpleasing': 902,\n",
       " 'devil-worshippers': 903,\n",
       " 'enterprise': 904,\n",
       " 'provision': 905,\n",
       " 'prospero': 907,\n",
       " 'stroll': 908,\n",
       " 'sciences': 155,\n",
       " 'attainable': 21325,\n",
       " 'market': 909,\n",
       " 'silhouetted': 910,\n",
       " 'reinforcing': 911,\n",
       " 'words': 912,\n",
       " 'himthenshe': 913,\n",
       " 'chucking': 3866,\n",
       " 'end-notes': 914,\n",
       " 'bunsen': 916,\n",
       " 'impresses': 919,\n",
       " 'hand-trolley': 918,\n",
       " 'visiting-cards': 920,\n",
       " 'kissed': 922,\n",
       " 'lets': 923,\n",
       " 'parrot-mouthed': 925,\n",
       " 'skug-cur': 926,\n",
       " 'dash-quick': 927,\n",
       " 'crucial': 928,\n",
       " 'biasing': 929,\n",
       " 'broken-off': 931,\n",
       " 'landed': 932,\n",
       " 'volcanism': 933,\n",
       " 'craned': 934,\n",
       " \"week's\": 161,\n",
       " 'toby': 936,\n",
       " 'good-sized': 138,\n",
       " 'winking': 937,\n",
       " 'conversations': 939,\n",
       " 'though': 940,\n",
       " 'schade': 942,\n",
       " 'vary': 943,\n",
       " 'extirpating': 944,\n",
       " \"vision's\": 945,\n",
       " 'which': 946,\n",
       " 'hindering': 947,\n",
       " 'hard-faced': 948,\n",
       " 'tremor': 949,\n",
       " 'drug': 951,\n",
       " 'nails': 952,\n",
       " 'discovering': 953,\n",
       " 'influences': 955,\n",
       " 'psychologists': 956,\n",
       " 'flaunting': 957,\n",
       " 'revealed': 958,\n",
       " 'they-think': 959,\n",
       " 'eruption': 960,\n",
       " 'age-blackened': 962,\n",
       " 'affrightedly': 963,\n",
       " 'pickman': 3877,\n",
       " 'temperance': 18786,\n",
       " 'confused': 964,\n",
       " 'climbed': 967,\n",
       " 'herding': 18789,\n",
       " 'flat-roofed': 968,\n",
       " 'toadstools': 969,\n",
       " 'nursed': 970,\n",
       " 'greenish': 971,\n",
       " 'cabinet': 972,\n",
       " 'perpetrate': 973,\n",
       " 'smiles': 974,\n",
       " 'principle’s': 975,\n",
       " 'thoughtful': 977,\n",
       " 'smiling': 978,\n",
       " 'apedom': 979,\n",
       " 'agape': 980,\n",
       " 'guyed': 11311,\n",
       " 'burglaries': 983,\n",
       " 'sardonicus': 984,\n",
       " 'efficient': 986,\n",
       " \"bachelor's\": 987,\n",
       " 'inequidistant': 988,\n",
       " 'lore': 989,\n",
       " 'ferrets': 990,\n",
       " 'scholar': 991,\n",
       " 'plenty-lots': 992,\n",
       " 'dovetailed': 993,\n",
       " 'obtain': 994,\n",
       " 'hoot': 995,\n",
       " 'robust': 996,\n",
       " 'endearment': 998,\n",
       " 'grew': 999,\n",
       " 'water-snake': 1000,\n",
       " 'indorse': 1001,\n",
       " 'individuality': 1002,\n",
       " 'well-grown': 1003,\n",
       " 'over-profound': 1004,\n",
       " 'genuine': 1005,\n",
       " 'connexion': 1006,\n",
       " 'leaned': 1007,\n",
       " 'chronicle': 1008,\n",
       " 'sustained': 1009,\n",
       " 'tanning': 1010,\n",
       " 'spending': 9693,\n",
       " 'goner': 8620,\n",
       " 'wiles': 1011,\n",
       " 'stars': 1012,\n",
       " 'hedge': 1013,\n",
       " 'youngest': 1014,\n",
       " 'snaps': 1015,\n",
       " 'shivering': 1016,\n",
       " 'exhalation': 1017,\n",
       " 'raided': 1018,\n",
       " 'applauding': 1019,\n",
       " 'augmented': 1020,\n",
       " \"tonga's\": 1022,\n",
       " 'im-which': 1023,\n",
       " 'reddish': 1024,\n",
       " 'devoured': 1025,\n",
       " \"tent's\": 1026,\n",
       " 'leg-weary': 1027,\n",
       " ...}"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "id_types = pre.create_dictionary(doc_tokens)\n",
    "doc_ids = pre.create_dictionary(doc_labels)\n",
    "id_types"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "#### Sparse BOW Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "sparse_bow = pre.create_sparse_bow(doc_labels, doc_tokens, id_types, doc_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>doc_id</th>\n",
       "      <th>token_id</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"30\" valign=\"top\">1</th>\n",
       "      <th>13639</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8195</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16388</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12289</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16392</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5463</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8204</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16397</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8206</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8207</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8208</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16407</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16409</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8219</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16413</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16414</th>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10928</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16418</th>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16419</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9565</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16433</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19642</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16440</th>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"30\" valign=\"top\">17</th>\n",
       "      <th>16300</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16301</th>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16303</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16305</th>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16307</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16308</th>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16321</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16323</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16324</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16374</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16333</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16336</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16339</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16340</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16342</th>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16343</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16344</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16346</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16350</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21840</th>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16357</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16358</th>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16360</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16361</th>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16362</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4364</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16377</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16378</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16383</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5461</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>56639 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                  0\n",
       "doc_id token_id    \n",
       "1      13639      2\n",
       "       8195       1\n",
       "       16388      1\n",
       "       6          1\n",
       "       12289      1\n",
       "       16392      1\n",
       "       9          1\n",
       "       5463       1\n",
       "       8204       1\n",
       "       16397      1\n",
       "       8206       1\n",
       "       8207       3\n",
       "       8208       1\n",
       "       18         2\n",
       "       16407      1\n",
       "       16409      1\n",
       "       8219       1\n",
       "       16413      1\n",
       "       16414      5\n",
       "       32         1\n",
       "       10928      1\n",
       "       16418     15\n",
       "       16419      2\n",
       "       9565       1\n",
       "       48         1\n",
       "       16433      1\n",
       "       50         4\n",
       "       51        27\n",
       "       19642      1\n",
       "       16440     19\n",
       "...              ..\n",
       "17     16300      1\n",
       "       16301      8\n",
       "       16303      3\n",
       "       16305      5\n",
       "       16307      1\n",
       "       16308      5\n",
       "       16321      2\n",
       "       16323      2\n",
       "       16324      1\n",
       "       16374      1\n",
       "       16333      2\n",
       "       16336      1\n",
       "       16339      2\n",
       "       16340      1\n",
       "       16342     11\n",
       "       16343      1\n",
       "       16344      1\n",
       "       16346      1\n",
       "       16350      1\n",
       "       21840     20\n",
       "       16357      2\n",
       "       16358      5\n",
       "       16360      1\n",
       "       16361     16\n",
       "       16362      1\n",
       "       4364       2\n",
       "       16377      1\n",
       "       16378      4\n",
       "       16383      4\n",
       "       5461       1\n",
       "\n",
       "[56639 rows x 1 columns]"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sparse_bow"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Topic Modeling with Gensim"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "##### Saving Sparse BOW"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "pre.save_sparse_bow(sparse_bow, \"gensim_txt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "#### Import Market Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO summa.preprocessing.cleaner: 'pattern' package not found; tag filters are not available for English\n"
     ]
    }
   ],
   "source": [
    "from gensim.models import LdaModel\n",
    "from gensim.corpora import MmCorpus\n",
    "from gensim.corpora import SvmLightCorpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO gensim.matutils: initializing corpus reader from gensim_txt.mm\n",
      "INFO gensim.matutils: accepted corpus with 17 documents, 22232 features, 369906 non-zero entries\n"
     ]
    }
   ],
   "source": [
    "mm = MmCorpus(\"gensim_txt.mm\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO gensim.matutils: initializing corpus reader from gensim_txt.mm\n",
      "INFO gensim.matutils: accepted corpus with 17 documents, 22232 features, 369906 non-zero entries\n",
      "INFO gensim.corpora.svmlightcorpus: converting corpus to SVMlight format: corpus.svmlight\n",
      "INFO gensim.corpora.indexedcorpus: saving SvmLightCorpus index to corpus.svmlight.index\n"
     ]
    }
   ],
   "source": [
    "corpus = MmCorpus('gensim_txt.mm')\n",
    "SvmLightCorpus.serialize('corpus.svmlight', corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "doc2id = {value : key for key, value in doc_ids.items()}\n",
    "type2id = {value : key for key, value in id_types.items()}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "##### Remove Features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "type2id = {value : key for key, value in id_types.items()}\n",
    "sparse_bow_collapsed = sparse_bow.groupby(sparse_bow.index.get_level_values('token_id')).sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "sparse_bow_hapax = sparse_bow_collapsed.loc[sparse_bow_collapsed[0] == 1]\n",
    "hapax = [type2id[key] for key in sparse_bow_hapax.index.get_level_values('token_id')]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "len(hapax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import os.path\n",
    "basepath = os.path.abspath('.')\n",
    "\n",
    "with open(os.path.join(basepath, \"tutorial_supplementals\", \"stopwords\", \"de.txt\"), 'r', encoding = 'utf-8') as f: \n",
    "    stopword_list = f.read().split('\\n')\n",
    "    \n",
    "stopword_list = set(stopword_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO preprocessing: Finding hapax legomena ...\n",
      "INFO preprocessing: Finding stopwords ...\n",
      "INFO preprocessing: Removing features ...\n"
     ]
    }
   ],
   "source": [
    "hapax_from_remove = pre.find_hapax(sparse_bow, id_types)\n",
    "stopwords_from_remove = pre.find_stopwords(sparse_bow, id_types, mfw=75)\n",
    "\n",
    "#features_to_be_removed = set(hapax_from_remove + stopwords_from_remove)\n",
    "features_to_be_removed = stopwords_from_remove\n",
    "\n",
    "sparse_bow_short = pre.remove_features(sparse_bow, id_types, features_to_be_removed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "75"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(features_to_be_removed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "pre.save_sparse_bow(sparse_bow_short, \"gensim_txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO gensim.matutils: initializing corpus reader from gensim_txt.mm\n",
      "INFO gensim.matutils: accepted corpus with 17 documents, 22232 features, 192053 non-zero entries\n"
     ]
    }
   ],
   "source": [
    "mm = MmCorpus(\"gensim_txt.mm\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "#### Convert sparse_bow to list of (doc, tokens) tuples (like doc2bow)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "doc2bow_list = []\n",
    "\n",
    "for doc in sparse_bow_short.index.groupby(sparse_bow_short.index.get_level_values('doc_id')):\n",
    "    temp = [(token, count) for token, count in zip(sparse_bow_short.loc[doc].index, sparse_bow_short.loc[doc][0])]\n",
    "    doc2bow_list.append(temp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Series([], Name: 0, dtype: int64)"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doc2bow_list[0]\n",
    "\n",
    "for"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO gensim.models.ldamodel: using symmetric alpha at 0.05\n",
      "INFO gensim.models.ldamodel: using symmetric eta at 4.49781855799937e-05\n",
      "INFO gensim.models.ldamodel: using serial LDA version on this node\n",
      "INFO gensim.models.ldamodel: running online LDA training, 20 topics, 10 passes over the supplied corpus of 17 documents, updating model once every 17 documents, evaluating perplexity every 17 documents, iterating 20x with a convergence threshold of 0.001000\n",
      "INFO gensim.models.ldamodel: -14.378 per-word bound, 21297.8 perplexity estimate based on a held-out corpus of 17 documents with 192053 words\n",
      "INFO gensim.models.ldamodel: PROGRESS: pass 0, at document #17/17\n",
      "INFO gensim.models.ldamodel: topic #2 (0.050): 0.003*\"wayfarers\" + 0.003*\"step-daughter\" + 0.003*\"grouping\" + 0.003*\"faster\" + 0.003*\"ape\" + 0.003*\"frenzied\" + 0.003*\"spies\" + 0.003*\"streaked\" + 0.003*\"loves\" + 0.003*\"rigging\"\n",
      "INFO gensim.models.ldamodel: topic #5 (0.050): 0.003*\"scuffling\" + 0.003*\"envisage\" + 0.003*\"ape\" + 0.003*\"artistry\" + 0.003*\"knockin\" + 0.003*\"frenzied\" + 0.003*\"born\" + 0.003*\"grouping\" + 0.003*\"here\" + 0.002*\"dispositions\"\n",
      "INFO gensim.models.ldamodel: topic #7 (0.050): 0.003*\"streaked\" + 0.003*\"envisage\" + 0.003*\"grouping\" + 0.003*\"artistry\" + 0.003*\"frenzied\" + 0.003*\"wayfarers\" + 0.003*\"ape\" + 0.003*\"faster\" + 0.003*\"concentric\" + 0.003*\"receives\"\n",
      "INFO gensim.models.ldamodel: topic #3 (0.050): 0.003*\"grouping\" + 0.003*\"frenzied\" + 0.003*\"faster\" + 0.003*\"free-born\" + 0.003*\"receives\" + 0.003*\"travelling\" + 0.003*\"loves\" + 0.003*\"spies\" + 0.002*\"leaders\" + 0.002*\"kneeling\"\n",
      "INFO gensim.models.ldamodel: topic #14 (0.050): 0.004*\"streaked\" + 0.003*\"scuffling\" + 0.003*\"knockin\" + 0.003*\"faster\" + 0.003*\"fossils\" + 0.003*\"twinkling\" + 0.003*\"grouping\" + 0.003*\"entrusted\" + 0.003*\"matkah\" + 0.003*\"ape\"\n",
      "INFO gensim.models.ldamodel: topic diff=5.872895, rho=1.000000\n",
      "INFO gensim.models.ldamodel: -10.937 per-word bound, 1960.7 perplexity estimate based on a held-out corpus of 17 documents with 192053 words\n",
      "INFO gensim.models.ldamodel: PROGRESS: pass 1, at document #17/17\n",
      "INFO gensim.models.ldamodel: topic #12 (0.050): 0.003*\"frenzied\" + 0.003*\"grouping\" + 0.003*\"scuffling\" + 0.003*\"streaked\" + 0.003*\"born\" + 0.003*\"blown\" + 0.003*\"ape\" + 0.003*\"wayfarers\" + 0.003*\"ac\" + 0.003*\"spies\"\n",
      "INFO gensim.models.ldamodel: topic #5 (0.050): 0.003*\"here\" + 0.003*\"born\" + 0.003*\"envisage\" + 0.003*\"knockin\" + 0.003*\"artistry\" + 0.002*\"scuffling\" + 0.002*\"we'll\" + 0.002*\"ape\" + 0.002*\"frenzied\" + 0.002*\"grouping\"\n",
      "INFO gensim.models.ldamodel: topic #2 (0.050): 0.003*\"grouping\" + 0.003*\"wayfarers\" + 0.003*\"step-daughter\" + 0.003*\"frenzied\" + 0.003*\"faster\" + 0.002*\"ape\" + 0.002*\"streaked\" + 0.002*\"spies\" + 0.002*\"loves\" + 0.002*\"versa\"\n",
      "INFO gensim.models.ldamodel: topic #6 (0.050): 0.005*\"caravan\" + 0.004*\"free-born\" + 0.004*\"half-sentient\" + 0.003*\"loves\" + 0.003*\"envisage\" + 0.003*\"versa\" + 0.003*\"spies\" + 0.003*\"faster\" + 0.003*\"concentric\" + 0.003*\"artistry\"\n",
      "INFO gensim.models.ldamodel: topic #19 (0.050): 0.003*\"blown\" + 0.003*\"scuffling\" + 0.003*\"knockin\" + 0.003*\"dome\" + 0.003*\"free-born\" + 0.002*\"tripods\" + 0.002*\"receives\" + 0.002*\"artistry\" + 0.002*\"wayfarers\" + 0.002*\"we'll\"\n",
      "INFO gensim.models.ldamodel: topic diff=2.620039, rho=0.577350\n",
      "INFO gensim.models.ldamodel: -9.721 per-word bound, 843.9 perplexity estimate based on a held-out corpus of 17 documents with 192053 words\n",
      "INFO gensim.models.ldamodel: PROGRESS: pass 2, at document #17/17\n",
      "INFO gensim.models.ldamodel: topic #18 (0.050): 0.008*\"scuffling\" + 0.005*\"blown\" + 0.005*\"frenzied\" + 0.004*\"prominent\" + 0.004*\"beady\" + 0.004*\"murderers\" + 0.004*\"laps\" + 0.004*\"streaked\" + 0.004*\"fathomed\" + 0.004*\"blunted\"\n",
      "INFO gensim.models.ldamodel: topic #7 (0.050): 0.006*\"fossils\" + 0.004*\"wayfarers\" + 0.003*\"ape\" + 0.003*\"artistry\" + 0.003*\"frenzied\" + 0.003*\"travelling\" + 0.003*\"spies\" + 0.003*\"step-daughter\" + 0.003*\"envisage\" + 0.003*\"dome\"\n",
      "INFO gensim.models.ldamodel: topic #2 (0.050): 0.002*\"grouping\" + 0.002*\"wayfarers\" + 0.002*\"step-daughter\" + 0.002*\"frenzied\" + 0.002*\"faster\" + 0.002*\"ape\" + 0.002*\"streaked\" + 0.002*\"spies\" + 0.002*\"loves\" + 0.002*\"versa\"\n",
      "INFO gensim.models.ldamodel: topic #11 (0.050): 0.009*\"paved\" + 0.007*\"spurs\" + 0.006*\"envisage\" + 0.006*\"accumulative\" + 0.005*\"knockin\" + 0.005*\"kozaki\" + 0.005*\"grossest\" + 0.005*\"we'll\" + 0.005*\"arms\" + 0.004*\"artistry\"\n",
      "INFO gensim.models.ldamodel: topic #16 (0.050): 0.003*\"drills\" + 0.003*\"scuffling\" + 0.003*\"frenzied\" + 0.003*\"grouping\" + 0.002*\"step-daughter\" + 0.002*\"blown\" + 0.002*\"knockin\" + 0.002*\"spies\" + 0.002*\"swearing\" + 0.002*\"prominent\"\n",
      "INFO gensim.models.ldamodel: topic diff=2.145682, rho=0.500000\n",
      "INFO gensim.models.ldamodel: -9.232 per-word bound, 601.2 perplexity estimate based on a held-out corpus of 17 documents with 192053 words\n",
      "INFO gensim.models.ldamodel: PROGRESS: pass 3, at document #17/17\n",
      "INFO gensim.models.ldamodel: topic #11 (0.050): 0.010*\"paved\" + 0.007*\"spurs\" + 0.007*\"accumulative\" + 0.007*\"envisage\" + 0.006*\"knockin\" + 0.006*\"grossest\" + 0.006*\"kozaki\" + 0.005*\"arms\" + 0.005*\"we'll\" + 0.005*\"naturalists\"\n",
      "INFO gensim.models.ldamodel: topic #3 (0.050): 0.002*\"grouping\" + 0.002*\"frenzied\" + 0.002*\"free-born\" + 0.002*\"receives\" + 0.002*\"faster\" + 0.002*\"scuffling\" + 0.002*\"travelling\" + 0.002*\"loves\" + 0.002*\"leaders\" + 0.002*\"knockin\"\n",
      "INFO gensim.models.ldamodel: topic #6 (0.050): 0.006*\"caravan\" + 0.004*\"half-sentient\" + 0.004*\"free-born\" + 0.004*\"versa\" + 0.004*\"artistry\" + 0.004*\"loves\" + 0.004*\"wasted\" + 0.003*\"concentric\" + 0.003*\"envisage\" + 0.003*\"commence\"\n",
      "INFO gensim.models.ldamodel: topic #15 (0.050): 0.007*\"akela's\" + 0.005*\"sick-dog\" + 0.005*\"se\" + 0.003*\"magistrate\" + 0.003*\"streaked\" + 0.003*\"pooh-pooh\" + 0.003*\"linked\" + 0.003*\"scuffling\" + 0.003*\"thickness\" + 0.003*\"ports\"\n",
      "INFO gensim.models.ldamodel: topic #1 (0.050): 0.002*\"knockin\" + 0.002*\"wayfarers\" + 0.002*\"frenzied\" + 0.002*\"grouping\" + 0.002*\"step-daughter\" + 0.002*\"skirting\" + 0.002*\"artistry\" + 0.001*\"streaked\" + 0.001*\"rigging\" + 0.001*\"fossils\"\n",
      "INFO gensim.models.ldamodel: topic diff=1.610747, rho=0.447214\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-22-09a2fb253791>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mLdaModel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcorpus\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mid2word\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtype2id\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnum_topics\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m20\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpasses\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0miterations\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m20\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/home/sina/.local/lib/python3.5/site-packages/gensim/models/ldamodel.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, corpus, num_topics, id2word, distributed, chunksize, passes, update_every, alpha, eta, decay, offset, eval_every, iterations, gamma_threshold, minimum_probability, random_state, ns_conf, minimum_phi_value, per_word_topics)\u001b[0m\n\u001b[1;32m    332\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mcorpus\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    333\u001b[0m             \u001b[0muse_numpy\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdispatcher\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 334\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcorpus\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchunks_as_numpy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0muse_numpy\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    335\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    336\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0minit_dir_prior\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprior\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/sina/.local/lib/python3.5/site-packages/gensim/models/ldamodel.py\u001b[0m in \u001b[0;36mupdate\u001b[0;34m(self, corpus, chunksize, decay, offset, passes, update_every, eval_every, iterations, gamma_threshold, chunks_as_numpy)\u001b[0m\n\u001b[1;32m    633\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    634\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0meval_every\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreallen\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mlencorpus\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchunk_no\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0meval_every\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumworkers\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 635\u001b[0;31m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlog_perplexity\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchunk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtotal_docs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlencorpus\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    636\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    637\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdispatcher\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/sina/.local/lib/python3.5/site-packages/gensim/models/ldamodel.py\u001b[0m in \u001b[0;36mlog_perplexity\u001b[0;34m(self, chunk, total_docs)\u001b[0m\n\u001b[1;32m    524\u001b[0m         \u001b[0mcorpus_words\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcnt\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mdocument\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mchunk\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcnt\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdocument\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    525\u001b[0m         \u001b[0msubsample_ratio\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m1.0\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mtotal_docs\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchunk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 526\u001b[0;31m         \u001b[0mperwordbound\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbound\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchunk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msubsample_ratio\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msubsample_ratio\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0msubsample_ratio\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mcorpus_words\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    527\u001b[0m         logger.info(\"%.3f per-word bound, %.1f perplexity estimate based on a held-out corpus of %i documents with %i words\" %\n\u001b[1;32m    528\u001b[0m                     (perwordbound, np.exp2(-perwordbound), len(chunk), corpus_words))\n",
      "\u001b[0;32m/home/sina/.local/lib/python3.5/site-packages/gensim/models/ldamodel.py\u001b[0m in \u001b[0;36mbound\u001b[0;34m(self, corpus, gamma, subsample_ratio)\u001b[0m\n\u001b[1;32m    731\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    732\u001b[0m             \u001b[0;31m# E[log p(doc | theta, beta)]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 733\u001b[0;31m             \u001b[0mscore\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcnt\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mlogsumexp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mElogthetad\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mElogbeta\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mid\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcnt\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdoc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    734\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    735\u001b[0m             \u001b[0;31m# E[log p(theta | alpha) - log q(theta | gamma)]; assumes alpha is a vector\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3/dist-packages/numpy/core/fromnumeric.py\u001b[0m in \u001b[0;36msum\u001b[0;34m(a, axis, dtype, out, keepdims)\u001b[0m\n\u001b[1;32m   1823\u001b[0m     \"\"\"\n\u001b[1;32m   1824\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_gentype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1825\u001b[0;31m         \u001b[0mres\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_sum_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1826\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mout\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1827\u001b[0m             \u001b[0mout\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m...\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mres\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/sina/.local/lib/python3.5/site-packages/gensim/models/ldamodel.py\u001b[0m in \u001b[0;36m<genexpr>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m    731\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    732\u001b[0m             \u001b[0;31m# E[log p(doc | theta, beta)]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 733\u001b[0;31m             \u001b[0mscore\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcnt\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mlogsumexp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mElogthetad\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mElogbeta\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mid\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcnt\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdoc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    734\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    735\u001b[0m             \u001b[0;31m# E[log p(theta | alpha) - log q(theta | gamma)]; assumes alpha is a vector\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/sina/.local/lib/python3.5/site-packages/scipy/special/_logsumexp.py\u001b[0m in \u001b[0;36mlogsumexp\u001b[0;34m(a, axis, b, keepdims, return_sign)\u001b[0m\n\u001b[1;32m     90\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     91\u001b[0m     \"\"\"\n\u001b[0;32m---> 92\u001b[0;31m     \u001b[0ma\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_asarray_validated\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcheck_finite\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     93\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mb\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     94\u001b[0m         \u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbroadcast_arrays\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/sina/.local/lib/python3.5/site-packages/scipy/_lib/_util.py\u001b[0m in \u001b[0;36m_asarray_validated\u001b[0;34m(a, check_finite, sparse_ok, objects_ok, mask_ok, as_inexact)\u001b[0m\n\u001b[1;32m    226\u001b[0m     \"\"\"\n\u001b[1;32m    227\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0msparse_ok\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 228\u001b[0;31m         \u001b[0;32mimport\u001b[0m \u001b[0mscipy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msparse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    229\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mscipy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msparse\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0missparse\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    230\u001b[0m             msg = ('Sparse matrices are not supported by this function. '\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "model = LdaModel(corpus=mm, id2word=type2id, num_topics=20, passes = 10, iterations = 20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "model.get_document_topics(doc2bow_list[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "#Anzeige der keywords für topic n\n",
    "n = 1\n",
    "topic_nr_x = model.get_topic_terms(n)\n",
    "\n",
    "topicTerms = [type2id[i[0]] for i in topic_nr_x]\n",
    "topicTerms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "#Topic probabilities - zum Testen der Funktion\n",
    "topics = model.show_topics(num_topics = model.num_topics, formatted=False)\n",
    "keywords = []\n",
    "\n",
    "num_topics = model.num_topics\n",
    "topics_df = pre.pd.DataFrame(index = range(num_topics), columns= range(10))\n",
    "\n",
    "for topic, values in topics:\n",
    "    keyword = [value[0] for value in values]\n",
    "    topics_df.loc[topic] = keyword\n",
    "\n",
    "topics_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "topics_df = pre.gensim2dataframe(model)\n",
    "topics_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "doc_topic = topics_df.T\n",
    "doc_topic"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Visualisierung "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "'''lda_model = 'out_easy/corpus.lda'\n",
    "corpus = 'out_easy/corpus.mm'\n",
    "dictionary = 'out_easy/corpus.dict'\n",
    "doc_labels = 'out_easy/corpus_doclabels.txt'\n",
    "interactive  = False\n",
    "\n",
    "vis = visual.Visualization(lda_model, corpus, dictionary, doc_labels, interactive)'''"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "##### Create a document-topic matrix (that is a pandas data frame actually)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "doc_topic = visual.create_doc_topic(mm, model, doc_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "##### Visualize document-topic matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "visual.doc_topic_heatmap(doc_topic)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "##### Visualize topic distribution in a single document"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "visual.plot_doc_topics(doc_topic, 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "items = model.show_topic(1)\n",
    "keywords = [item[0] for item in items]\n",
    "keywords\n",
    "items = model.show_topic(1)\n",
    "propabilities = [item[1] for item in items]\n",
    "propabilities\n",
    "items"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "topic_nr = 3\n",
    "number_of_keywords = 10\n",
    "visual.show_wordle_for_topic(model, topic_nr, number_of_keywords)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "source": [
    "![success](http://cdn2.hubspot.net/hub/128506/file-446943132-jpg/images/computer_woman_success.jpg)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}