IntegrationTest_v01.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\philip\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages\\gensim\\utils.py:855: UserWarning: detected Windows; aliasing chunkize to chunkize_serial\n",
      "  warnings.warn(\"detected Windows; aliasing chunkize to chunkize_serial\")\n",
      "c:\\users\\philip\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages\\funcy\\decorators.py:56: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead\n",
      "  spec = inspect.getargspec(func)\n",
      "c:\\users\\philip\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages\\funcy\\decorators.py:56: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead\n",
      "  spec = inspect.getargspec(func)\n",
      "c:\\users\\philip\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages\\funcy\\decorators.py:56: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead\n",
      "  spec = inspect.getargspec(func)\n",
      "c:\\users\\philip\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages\\funcy\\decorators.py:56: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead\n",
      "  spec = inspect.getargspec(func)\n",
      "c:\\users\\philip\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages\\funcy\\decorators.py:56: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead\n",
      "  spec = inspect.getargspec(func)\n",
      "c:\\users\\philip\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages\\funcy\\decorators.py:56: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead\n",
      "  spec = inspect.getargspec(func)\n"
     ]
    }
   ],
   "source": [
    "from dariah_topics import preprocessing as pre\n",
    "from dariah_topics import visualization as visual\n",
    "from dariah_topics import mallet as mal\n",
    "# Warning is Gensim related"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Liste mit Dateinamen erzeugen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\philip\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages\\IPython\\core\\formatters.py:92: DeprecationWarning: DisplayFormatter._ipython_display_formatter_default is deprecated: use @default decorator instead.\n",
      "  def _ipython_display_formatter_default(self):\n",
      "c:\\users\\philip\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages\\IPython\\core\\formatters.py:98: DeprecationWarning: DisplayFormatter._formatters_default is deprecated: use @default decorator instead.\n",
      "  def _formatters_default(self):\n",
      "c:\\users\\philip\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages\\IPython\\core\\formatters.py:677: DeprecationWarning: PlainTextFormatter._deferred_printers_default is deprecated: use @default decorator instead.\n",
      "  def _deferred_printers_default(self):\n",
      "c:\\users\\philip\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages\\IPython\\core\\formatters.py:669: DeprecationWarning: PlainTextFormatter._singleton_printers_default is deprecated: use @default decorator instead.\n",
      "  def _singleton_printers_default(self):\n",
      "c:\\users\\philip\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages\\IPython\\core\\formatters.py:672: DeprecationWarning: PlainTextFormatter._type_printers_default is deprecated: use @default decorator instead.\n",
      "  def _type_printers_default(self):\n",
      "c:\\users\\philip\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages\\IPython\\core\\formatters.py:669: DeprecationWarning: PlainTextFormatter._singleton_printers_default is deprecated: use @default decorator instead.\n",
      "  def _singleton_printers_default(self):\n",
      "c:\\users\\philip\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages\\IPython\\core\\formatters.py:672: DeprecationWarning: PlainTextFormatter._type_printers_default is deprecated: use @default decorator instead.\n",
      "  def _type_printers_default(self):\n",
      "c:\\users\\philip\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages\\IPython\\core\\formatters.py:677: DeprecationWarning: PlainTextFormatter._deferred_printers_default is deprecated: use @default decorator instead.\n",
      "  def _deferred_printers_default(self):\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['corpus_txt\\\\Doyle_AScandalinBohemia.txt',\n",
       " 'corpus_txt\\\\Doyle_AStudyinScarlet.txt',\n",
       " 'corpus_txt\\\\Doyle_TheHoundoftheBaskervilles.txt',\n",
       " 'corpus_txt\\\\Doyle_TheSignoftheFour.txt',\n",
       " 'corpus_txt\\\\Howard_GodsoftheNorth.txt']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path_txt = \"corpus_txt\"\n",
    "#path_txt = \"grenzbote_plain/*/\"\n",
    "\n",
    "doclist_txt = pre.create_document_list(path_txt)\n",
    "doclist_txt[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['corpus_csv\\\\Doyle_AStudyinScarlet.txt.csv',\n",
       " 'corpus_csv\\\\Doyle_TheHoundoftheBaskervilles.txt.csv',\n",
       " 'corpus_csv\\\\Doyle_TheSignoftheFour.txt.csv',\n",
       " 'corpus_csv\\\\Howard_GodsoftheNorth.txt.csv',\n",
       " 'corpus_csv\\\\Howard_SchadowsinZamboula.txt.csv']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path_csv = \"corpus_csv\"\n",
    "\n",
    "doclist_csv = pre.create_document_list(path_csv, 'csv')\n",
    "doclist_csv[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#####  Liste mit Dokumentenlabels erzeugen - (Funktion wird durch Thorsten's generischere Funktion ersetzt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['corpus_txt\\\\Doyle_AScandalinBohemia.txt',\n",
       " 'corpus_txt\\\\Doyle_AStudyinScarlet.txt',\n",
       " 'corpus_txt\\\\Doyle_TheHoundoftheBaskervilles.txt',\n",
       " 'corpus_txt\\\\Doyle_TheSignoftheFour.txt',\n",
       " 'corpus_txt\\\\Howard_GodsoftheNorth.txt']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doc_labels = list(pre.get_labels(doclist_txt))\n",
    "doc_labels[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Corpus laden"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "corpus_txt = pre.read_from_txt(doclist_txt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "corpus_csv = pre.read_from_csv(doclist_csv)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Tokenisieren"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['a',\n",
       "  'scandal',\n",
       "  'in',\n",
       "  'bohemia',\n",
       "  'a',\n",
       "  'conan',\n",
       "  'doyle',\n",
       "  'i',\n",
       "  'to',\n",
       "  'sherlock',\n",
       "  'holmes',\n",
       "  'she',\n",
       "  'is',\n",
       "  'always',\n",
       "  'the',\n",
       "  'woman',\n",
       "  'i',\n",
       "  'have',\n",
       "  'seldom',\n",
       "  'heard',\n",
       "  'him',\n",
       "  'mention',\n",
       "  'her',\n",
       "  'under',\n",
       "  'any',\n",
       "  'other',\n",
       "  'name',\n",
       "  'in',\n",
       "  'his',\n",
       "  'eyes',\n",
       "  'she',\n",
       "  'eclipses',\n",
       "  'and',\n",
       "  'predominates',\n",
       "  'the',\n",
       "  'whole',\n",
       "  'of',\n",
       "  'her',\n",
       "  'sex',\n",
       "  'it',\n",
       "  'was',\n",
       "  'not',\n",
       "  'that',\n",
       "  'he',\n",
       "  'felt',\n",
       "  'any',\n",
       "  'emotion',\n",
       "  'akin',\n",
       "  'to',\n",
       "  'love',\n",
       "  'for',\n",
       "  'irene',\n",
       "  'adler',\n",
       "  'all',\n",
       "  'emotions',\n",
       "  'and',\n",
       "  'that',\n",
       "  'one',\n",
       "  'particularly',\n",
       "  'were',\n",
       "  'abhorrent',\n",
       "  'to',\n",
       "  'his',\n",
       "  'cold',\n",
       "  'precise',\n",
       "  'but',\n",
       "  'admirably',\n",
       "  'balanced',\n",
       "  'mind',\n",
       "  'he',\n",
       "  'was',\n",
       "  'i',\n",
       "  'take',\n",
       "  'it',\n",
       "  'the',\n",
       "  'most',\n",
       "  'perfect',\n",
       "  'reasoning',\n",
       "  'and',\n",
       "  'observing',\n",
       "  'machine',\n",
       "  'that',\n",
       "  'the',\n",
       "  'world',\n",
       "  'has',\n",
       "  'seen',\n",
       "  'but',\n",
       "  'as',\n",
       "  'a',\n",
       "  'lover',\n",
       "  'he',\n",
       "  'would',\n",
       "  'have',\n",
       "  'placed',\n",
       "  'himself',\n",
       "  'in',\n",
       "  'a',\n",
       "  'false',\n",
       "  'position',\n",
       "  'he',\n",
       "  'never',\n",
       "  'spoke',\n",
       "  'of',\n",
       "  'the',\n",
       "  'softer',\n",
       "  'passions',\n",
       "  'save',\n",
       "  'with',\n",
       "  'a',\n",
       "  'gibe',\n",
       "  'and',\n",
       "  'a',\n",
       "  'sneer',\n",
       "  'they',\n",
       "  'were',\n",
       "  'admirable',\n",
       "  'things',\n",
       "  'for',\n",
       "  'the',\n",
       "  'observer--excellent',\n",
       "  'for',\n",
       "  'drawing',\n",
       "  'the',\n",
       "  'veil',\n",
       "  'from',\n",
       "  \"men's\",\n",
       "  'motives',\n",
       "  'and',\n",
       "  'actions',\n",
       "  'but',\n",
       "  'for',\n",
       "  'the',\n",
       "  'trained',\n",
       "  'reasoner',\n",
       "  'to',\n",
       "  'admit',\n",
       "  'such',\n",
       "  'intrusions',\n",
       "  'into',\n",
       "  'his',\n",
       "  'own',\n",
       "  'delicate',\n",
       "  'and',\n",
       "  'finely',\n",
       "  'adjusted',\n",
       "  'temperament',\n",
       "  'was',\n",
       "  'to',\n",
       "  'introduce',\n",
       "  'a',\n",
       "  'distracting',\n",
       "  'factor',\n",
       "  'which',\n",
       "  'might',\n",
       "  'throw',\n",
       "  'a',\n",
       "  'doubt',\n",
       "  'upon',\n",
       "  'all',\n",
       "  'his',\n",
       "  'mental',\n",
       "  'results',\n",
       "  'grit',\n",
       "  'in',\n",
       "  'a',\n",
       "  'sensitive',\n",
       "  'instrument',\n",
       "  'or',\n",
       "  'a',\n",
       "  'crack',\n",
       "  'in',\n",
       "  'one',\n",
       "  'of',\n",
       "  'his',\n",
       "  'own',\n",
       "  'high-power',\n",
       "  'lenses',\n",
       "  'would',\n",
       "  'not',\n",
       "  'be',\n",
       "  'more',\n",
       "  'disturbing',\n",
       "  'than',\n",
       "  'a',\n",
       "  'strong',\n",
       "  'emotion',\n",
       "  'in',\n",
       "  'a',\n",
       "  'nature',\n",
       "  'such',\n",
       "  'as',\n",
       "  'his',\n",
       "  'and',\n",
       "  'yet',\n",
       "  'there',\n",
       "  'was',\n",
       "  'but',\n",
       "  'one',\n",
       "  'woman',\n",
       "  'to',\n",
       "  'him',\n",
       "  'and',\n",
       "  'that',\n",
       "  'woman',\n",
       "  'was',\n",
       "  'the',\n",
       "  'late',\n",
       "  'irene',\n",
       "  'adler',\n",
       "  'of',\n",
       "  'dubious',\n",
       "  'and',\n",
       "  'questionable',\n",
       "  'memory',\n",
       "  'i',\n",
       "  'had',\n",
       "  'seen',\n",
       "  'little',\n",
       "  'of',\n",
       "  'holmes',\n",
       "  'lately',\n",
       "  'my',\n",
       "  'marriage',\n",
       "  'had',\n",
       "  'drifted',\n",
       "  'us',\n",
       "  'away',\n",
       "  'from',\n",
       "  'each',\n",
       "  'other',\n",
       "  'my',\n",
       "  'own',\n",
       "  'complete',\n",
       "  'happiness',\n",
       "  'and',\n",
       "  'the',\n",
       "  'home-centred',\n",
       "  'interests',\n",
       "  'which',\n",
       "  'rise',\n",
       "  'up',\n",
       "  'around',\n",
       "  'the',\n",
       "  'man',\n",
       "  'who',\n",
       "  'first',\n",
       "  'finds',\n",
       "  'himself',\n",
       "  'master',\n",
       "  'of',\n",
       "  'his',\n",
       "  'own',\n",
       "  'establishment',\n",
       "  'were',\n",
       "  'sufficient',\n",
       "  'to',\n",
       "  'absorb',\n",
       "  'all',\n",
       "  'my',\n",
       "  'attention',\n",
       "  'while',\n",
       "  'holmes',\n",
       "  'who',\n",
       "  'loathed',\n",
       "  'every',\n",
       "  'form',\n",
       "  'of',\n",
       "  'society',\n",
       "  'with',\n",
       "  'his',\n",
       "  'whole',\n",
       "  'bohemian',\n",
       "  'soul',\n",
       "  'remained',\n",
       "  'in',\n",
       "  'our',\n",
       "  'lodgings',\n",
       "  'in',\n",
       "  'baker',\n",
       "  'street',\n",
       "  'buried',\n",
       "  'among',\n",
       "  'his',\n",
       "  'old',\n",
       "  'books',\n",
       "  'and',\n",
       "  'alternating',\n",
       "  'from',\n",
       "  'week',\n",
       "  'to',\n",
       "  'week',\n",
       "  'between',\n",
       "  'cocaine',\n",
       "  'and',\n",
       "  'ambition',\n",
       "  'the',\n",
       "  'drowsiness',\n",
       "  'of',\n",
       "  'the',\n",
       "  'drug',\n",
       "  'and',\n",
       "  'the',\n",
       "  'fierce',\n",
       "  'energy',\n",
       "  'of',\n",
       "  'his',\n",
       "  'own',\n",
       "  'keen',\n",
       "  'nature',\n",
       "  'he',\n",
       "  'was',\n",
       "  'still',\n",
       "  'as',\n",
       "  'ever',\n",
       "  'deeply',\n",
       "  'attracted',\n",
       "  'by',\n",
       "  'the',\n",
       "  'study',\n",
       "  'of',\n",
       "  'crime',\n",
       "  'and',\n",
       "  'occupied',\n",
       "  'his',\n",
       "  'immense',\n",
       "  'faculties',\n",
       "  'and',\n",
       "  'extraordinary',\n",
       "  'powers',\n",
       "  'of',\n",
       "  'observation',\n",
       "  'in',\n",
       "  'following',\n",
       "  'out',\n",
       "  'those',\n",
       "  'clews',\n",
       "  'and',\n",
       "  'clearing',\n",
       "  'up',\n",
       "  'those',\n",
       "  'mysteries',\n",
       "  'which',\n",
       "  'had',\n",
       "  'been',\n",
       "  'abandoned',\n",
       "  'as',\n",
       "  'hopeless',\n",
       "  'by',\n",
       "  'the',\n",
       "  'official',\n",
       "  'police',\n",
       "  'from',\n",
       "  'time',\n",
       "  'to',\n",
       "  'time',\n",
       "  'i',\n",
       "  'heard',\n",
       "  'some',\n",
       "  'vague',\n",
       "  'account',\n",
       "  'of',\n",
       "  'his',\n",
       "  'doings',\n",
       "  'of',\n",
       "  'his',\n",
       "  'summons',\n",
       "  'to',\n",
       "  'odessa',\n",
       "  'in',\n",
       "  'the',\n",
       "  'case',\n",
       "  'of',\n",
       "  'the',\n",
       "  'trepoff',\n",
       "  'murder',\n",
       "  'of',\n",
       "  'his',\n",
       "  'clearing',\n",
       "  'up',\n",
       "  'of',\n",
       "  'the',\n",
       "  'singular',\n",
       "  'tragedy',\n",
       "  'of',\n",
       "  'the',\n",
       "  'atkinson',\n",
       "  'brothers',\n",
       "  'at',\n",
       "  'trincomalee',\n",
       "  'and',\n",
       "  'finally',\n",
       "  'of',\n",
       "  'the',\n",
       "  'mission',\n",
       "  'which',\n",
       "  'he',\n",
       "  'had',\n",
       "  'accomplished',\n",
       "  'so',\n",
       "  'delicately',\n",
       "  'and',\n",
       "  'successfully',\n",
       "  'for',\n",
       "  'the',\n",
       "  'reigning',\n",
       "  'family',\n",
       "  'of',\n",
       "  'holland',\n",
       "  'beyond',\n",
       "  'these',\n",
       "  'signs',\n",
       "  'of',\n",
       "  'his',\n",
       "  'activity',\n",
       "  'however',\n",
       "  'which',\n",
       "  'i',\n",
       "  'merely',\n",
       "  'shared',\n",
       "  'with',\n",
       "  'all',\n",
       "  'the',\n",
       "  'readers',\n",
       "  'of',\n",
       "  'the',\n",
       "  'daily',\n",
       "  'press',\n",
       "  'i',\n",
       "  'knew',\n",
       "  'little',\n",
       "  'of',\n",
       "  'my',\n",
       "  'former',\n",
       "  'friend',\n",
       "  'and',\n",
       "  'companion',\n",
       "  'one',\n",
       "  'night--it',\n",
       "  'was',\n",
       "  'on',\n",
       "  'the',\n",
       "  'th',\n",
       "  'of',\n",
       "  'march',\n",
       "  'i',\n",
       "  'was',\n",
       "  'returning',\n",
       "  'from',\n",
       "  'a',\n",
       "  'journey',\n",
       "  'to',\n",
       "  'a',\n",
       "  'patient',\n",
       "  'for',\n",
       "  'i',\n",
       "  'had',\n",
       "  'now',\n",
       "  'returned',\n",
       "  'to',\n",
       "  'civil',\n",
       "  'practice',\n",
       "  'when',\n",
       "  'my',\n",
       "  'way',\n",
       "  'led',\n",
       "  'me',\n",
       "  'through',\n",
       "  'baker',\n",
       "  'street',\n",
       "  'as',\n",
       "  'i',\n",
       "  'passed',\n",
       "  'the',\n",
       "  'well-remembered',\n",
       "  'door',\n",
       "  'which',\n",
       "  'must',\n",
       "  'always',\n",
       "  'be',\n",
       "  'associated',\n",
       "  'in',\n",
       "  'my',\n",
       "  'mind',\n",
       "  'with',\n",
       "  'my',\n",
       "  'wooing',\n",
       "  'and',\n",
       "  'with',\n",
       "  'the',\n",
       "  'dark',\n",
       "  'incidents',\n",
       "  'of',\n",
       "  'the',\n",
       "  'study',\n",
       "  'in',\n",
       "  'scarlet',\n",
       "  'i',\n",
       "  'was',\n",
       "  'seized',\n",
       "  'with',\n",
       "  'a',\n",
       "  'keen',\n",
       "  'desire',\n",
       "  'to',\n",
       "  'see',\n",
       "  'holmes',\n",
       "  'again',\n",
       "  'and',\n",
       "  'to',\n",
       "  'know',\n",
       "  'how',\n",
       "  'he',\n",
       "  'was',\n",
       "  'employing',\n",
       "  'his',\n",
       "  'extraordinary',\n",
       "  'powers',\n",
       "  'his',\n",
       "  'rooms',\n",
       "  'were',\n",
       "  'brilliantly',\n",
       "  'lighted',\n",
       "  'and',\n",
       "  'even',\n",
       "  'as',\n",
       "  'i',\n",
       "  'looked',\n",
       "  'up',\n",
       "  'i',\n",
       "  'saw',\n",
       "  'his',\n",
       "  'tall',\n",
       "  'spare',\n",
       "  'figure',\n",
       "  'pass',\n",
       "  'twice',\n",
       "  'in',\n",
       "  'a',\n",
       "  'dark',\n",
       "  'silhouette',\n",
       "  'against',\n",
       "  'the',\n",
       "  'blind',\n",
       "  'he',\n",
       "  'was',\n",
       "  'pacing',\n",
       "  'the',\n",
       "  'room',\n",
       "  'swiftly',\n",
       "  'eagerly',\n",
       "  'with',\n",
       "  'his',\n",
       "  'head',\n",
       "  'sunk',\n",
       "  'upon',\n",
       "  'his',\n",
       "  'chest',\n",
       "  'and',\n",
       "  'his',\n",
       "  'hands',\n",
       "  'clasped',\n",
       "  'behind',\n",
       "  'him',\n",
       "  'to',\n",
       "  'me',\n",
       "  'who',\n",
       "  'knew',\n",
       "  'his',\n",
       "  'every',\n",
       "  'mood',\n",
       "  'and',\n",
       "  'habit',\n",
       "  'his',\n",
       "  'attitude',\n",
       "  'and',\n",
       "  'manner',\n",
       "  'told',\n",
       "  'their',\n",
       "  'own',\n",
       "  'story',\n",
       "  'he',\n",
       "  'was',\n",
       "  'at',\n",
       "  'work',\n",
       "  'again',\n",
       "  'he',\n",
       "  'had',\n",
       "  'risen',\n",
       "  'out',\n",
       "  'of',\n",
       "  'his',\n",
       "  'drug-created',\n",
       "  'dreams',\n",
       "  'and',\n",
       "  'was',\n",
       "  'hot',\n",
       "  'upon',\n",
       "  'the',\n",
       "  'scent',\n",
       "  'of',\n",
       "  'some',\n",
       "  'new',\n",
       "  'problem',\n",
       "  'i',\n",
       "  'rang',\n",
       "  'the',\n",
       "  'bell',\n",
       "  'and',\n",
       "  'was',\n",
       "  'shown',\n",
       "  'up',\n",
       "  'to',\n",
       "  'the',\n",
       "  'chamber',\n",
       "  'which',\n",
       "  'had',\n",
       "  'formerly',\n",
       "  'been',\n",
       "  'in',\n",
       "  'part',\n",
       "  'my',\n",
       "  'own',\n",
       "  'his',\n",
       "  'manner',\n",
       "  'was',\n",
       "  'not',\n",
       "  'effusive',\n",
       "  'it',\n",
       "  'seldom',\n",
       "  'was',\n",
       "  'but',\n",
       "  'he',\n",
       "  'was',\n",
       "  'glad',\n",
       "  'i',\n",
       "  'think',\n",
       "  'to',\n",
       "  'see',\n",
       "  'me',\n",
       "  'with',\n",
       "  'hardly',\n",
       "  'a',\n",
       "  'word',\n",
       "  'spoken',\n",
       "  'but',\n",
       "  'with',\n",
       "  'a',\n",
       "  'kindly',\n",
       "  'eye',\n",
       "  'he',\n",
       "  'waved',\n",
       "  'me',\n",
       "  'to',\n",
       "  'an',\n",
       "  'armchair',\n",
       "  'threw',\n",
       "  'across',\n",
       "  'his',\n",
       "  'case',\n",
       "  'of',\n",
       "  'cigars',\n",
       "  'and',\n",
       "  'indicated',\n",
       "  'a',\n",
       "  'spirit',\n",
       "  'case',\n",
       "  'and',\n",
       "  'a',\n",
       "  'gasogene',\n",
       "  'in',\n",
       "  'the',\n",
       "  'corner',\n",
       "  'then',\n",
       "  'he',\n",
       "  'stood',\n",
       "  'before',\n",
       "  'the',\n",
       "  'fire',\n",
       "  'and',\n",
       "  'looked',\n",
       "  'me',\n",
       "  'over',\n",
       "  'in',\n",
       "  'his',\n",
       "  'singular',\n",
       "  'introspective',\n",
       "  'fashion',\n",
       "  'wedlock',\n",
       "  'suits',\n",
       "  'you',\n",
       "  'he',\n",
       "  'remarked',\n",
       "  'i',\n",
       "  'think',\n",
       "  'watson',\n",
       "  'that',\n",
       "  'you',\n",
       "  'have',\n",
       "  'put',\n",
       "  'on',\n",
       "  'seven',\n",
       "  'and',\n",
       "  'a',\n",
       "  'half',\n",
       "  'pounds',\n",
       "  'since',\n",
       "  'i',\n",
       "  'saw',\n",
       "  'you',\n",
       "  'seven',\n",
       "  'i',\n",
       "  'answered',\n",
       "  'indeed',\n",
       "  'i',\n",
       "  'should',\n",
       "  'have',\n",
       "  'thought',\n",
       "  'a',\n",
       "  'little',\n",
       "  'more',\n",
       "  'just',\n",
       "  'a',\n",
       "  'trifle',\n",
       "  'more',\n",
       "  'i',\n",
       "  'fancy',\n",
       "  'watson',\n",
       "  'and',\n",
       "  'in',\n",
       "  'practice',\n",
       "  'again',\n",
       "  'i',\n",
       "  'observe',\n",
       "  'you',\n",
       "  'did',\n",
       "  'not',\n",
       "  'tell',\n",
       "  'me',\n",
       "  'that',\n",
       "  'you',\n",
       "  'intended',\n",
       "  'to',\n",
       "  'go',\n",
       "  'into',\n",
       "  'harness',\n",
       "  'then',\n",
       "  'how',\n",
       "  'do',\n",
       "  'you',\n",
       "  'know',\n",
       "  'i',\n",
       "  'see',\n",
       "  'it',\n",
       "  'i',\n",
       "  'deduce',\n",
       "  'it',\n",
       "  'how',\n",
       "  'do',\n",
       "  'i',\n",
       "  'know',\n",
       "  'that',\n",
       "  'you',\n",
       "  'have',\n",
       "  'been',\n",
       "  'getting',\n",
       "  'yourself',\n",
       "  'very',\n",
       "  'wet',\n",
       "  'lately',\n",
       "  'and',\n",
       "  'that',\n",
       "  'you',\n",
       "  'have',\n",
       "  'a',\n",
       "  'most',\n",
       "  'clumsy',\n",
       "  'and',\n",
       "  'careless',\n",
       "  'servant',\n",
       "  'girl',\n",
       "  'my',\n",
       "  'dear',\n",
       "  'holmes',\n",
       "  'said',\n",
       "  'i',\n",
       "  'this',\n",
       "  'is',\n",
       "  'too',\n",
       "  'much',\n",
       "  'you',\n",
       "  'would',\n",
       "  'certainly',\n",
       "  'have',\n",
       "  'been',\n",
       "  'burned',\n",
       "  'had',\n",
       "  'you',\n",
       "  'lived',\n",
       "  'a',\n",
       "  'few',\n",
       "  'centuries',\n",
       "  'ago',\n",
       "  'it',\n",
       "  'is',\n",
       "  'true',\n",
       "  'that',\n",
       "  'i',\n",
       "  'had',\n",
       "  'a',\n",
       "  'country',\n",
       "  'walk',\n",
       "  'on',\n",
       "  'thursday',\n",
       "  'and',\n",
       "  'came',\n",
       "  'home',\n",
       "  'in',\n",
       "  'a',\n",
       "  'dreadful',\n",
       "  'mess',\n",
       "  'but',\n",
       "  'as',\n",
       "  'i',\n",
       "  'have',\n",
       "  'changed',\n",
       "  'my',\n",
       "  'clothes',\n",
       "  'i',\n",
       "  \"can't\",\n",
       "  'imagine',\n",
       "  'how',\n",
       "  'you',\n",
       "  'deduce',\n",
       "  'it',\n",
       "  'as',\n",
       "  'to',\n",
       "  'mary',\n",
       "  'jane',\n",
       "  'she',\n",
       "  'is',\n",
       "  'incorrigible',\n",
       "  'and',\n",
       "  'my',\n",
       "  'wife',\n",
       "  'has',\n",
       "  'given',\n",
       "  'her',\n",
       "  'notice',\n",
       "  'but',\n",
       "  'there',\n",
       "  'again',\n",
       "  'i',\n",
       "  'fail',\n",
       "  'to',\n",
       "  'see',\n",
       "  'how',\n",
       "  'you',\n",
       "  'work',\n",
       "  'it',\n",
       "  'out',\n",
       "  'he',\n",
       "  'chuckled',\n",
       "  'to',\n",
       "  'himself',\n",
       "  'and',\n",
       "  'rubbed',\n",
       "  'his',\n",
       "  'long',\n",
       "  'nervous',\n",
       "  'hands',\n",
       "  'together',\n",
       "  'it',\n",
       "  'is',\n",
       "  'simplicity',\n",
       "  'itself',\n",
       "  'said',\n",
       "  'he',\n",
       "  'my',\n",
       "  'eyes',\n",
       "  'tell',\n",
       "  'me',\n",
       "  'that',\n",
       "  'on',\n",
       "  'the',\n",
       "  'inside',\n",
       "  'of',\n",
       "  'your',\n",
       "  'left',\n",
       "  'shoe',\n",
       "  'just',\n",
       "  'where',\n",
       "  'the',\n",
       "  'firelight',\n",
       "  'strikes',\n",
       "  'it',\n",
       "  'the',\n",
       "  'leather',\n",
       "  'is',\n",
       "  'scored',\n",
       "  'by',\n",
       "  'six',\n",
       "  'almost',\n",
       "  'parallel',\n",
       "  'cuts',\n",
       "  'obviously',\n",
       "  'they',\n",
       "  'have',\n",
       "  'been',\n",
       "  'caused',\n",
       "  'by',\n",
       "  'someone',\n",
       "  'who',\n",
       "  'has',\n",
       "  'very',\n",
       "  'carelessly',\n",
       "  'scraped',\n",
       "  'round',\n",
       "  'the',\n",
       "  'edges',\n",
       "  'of',\n",
       "  'the',\n",
       "  'sole',\n",
       "  'in',\n",
       "  'order',\n",
       "  'to',\n",
       "  'remove',\n",
       "  'crusted',\n",
       "  'mud',\n",
       "  'from',\n",
       "  'it',\n",
       "  'hence',\n",
       "  'you',\n",
       "  'see',\n",
       "  'my',\n",
       "  'double',\n",
       "  'deduction',\n",
       "  'that',\n",
       "  'you',\n",
       "  'had',\n",
       "  'been',\n",
       "  'out',\n",
       "  'in',\n",
       "  'vile',\n",
       "  'weather',\n",
       "  'and',\n",
       "  'that',\n",
       "  'you',\n",
       "  'had',\n",
       "  'a',\n",
       "  'particularly',\n",
       "  'malignant',\n",
       "  'boot-slicking',\n",
       "  'specimen',\n",
       "  'of',\n",
       "  'the',\n",
       "  'london',\n",
       "  'slavey',\n",
       "  'as',\n",
       "  'to',\n",
       "  'your',\n",
       "  'practice',\n",
       "  'if',\n",
       "  'a',\n",
       "  'gentleman',\n",
       "  'walks',\n",
       "  'into',\n",
       "  'my',\n",
       "  'rooms',\n",
       "  'smelling',\n",
       "  'of',\n",
       "  'iodoform',\n",
       "  'with',\n",
       "  'a',\n",
       "  'black',\n",
       "  'mark',\n",
       "  'of',\n",
       "  'nitrate',\n",
       "  'of',\n",
       "  'silver',\n",
       "  'upon',\n",
       "  'his',\n",
       "  'right',\n",
       "  'forefinger',\n",
       "  'and',\n",
       "  'a',\n",
       "  'bulge',\n",
       "  'on',\n",
       "  'the',\n",
       "  'side',\n",
       "  'of',\n",
       "  'his',\n",
       "  'top',\n",
       "  'hat',\n",
       "  'to',\n",
       "  'show',\n",
       "  'where',\n",
       "  'he',\n",
       "  'has',\n",
       "  'secreted',\n",
       "  'his',\n",
       "  ...]]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doc_tokens = [list(pre.tokenize(txt)) for txt in list(corpus_txt)]\n",
    "doc_tokens[:1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Create Dictionaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "id_types, doc_ids = pre.create_dictionaries(doc_labels, doc_tokens)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Sparse BOW Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sparse_bow = pre.create_mm(doc_labels, doc_tokens, id_types, doc_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>doc_id</th>\n",
       "      <th>token_id</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"30\" valign=\"top\">0</th>\n",
       "      <th>18432</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8195</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5462</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16390</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10924</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4109</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6829</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16403</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8212</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16405</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8214</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17753</th>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16408</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8217</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4123</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4124</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12320</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4129</th>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16418</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9563</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12294</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19121</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20520</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4139</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16428</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20526</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12335</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6152</th>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"30\" valign=\"top\">16</th>\n",
       "      <th>20414</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4031</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24512</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5451</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16324</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4086</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20424</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24521</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6954</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20430</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16638</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4050</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16339</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16340</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16343</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8154</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20446</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12256</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8161</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16354</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8165</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12266</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16363</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4076</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16365</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12275</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16374</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12280</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4089</th>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16382</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>58745 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                  0\n",
       "doc_id token_id    \n",
       "0      18432      1\n",
       "       8195       1\n",
       "       5462       1\n",
       "       16390      1\n",
       "       10924      1\n",
       "       12         1\n",
       "       4109       1\n",
       "       6829       1\n",
       "       16403      1\n",
       "       8212       1\n",
       "       16405      1\n",
       "       8214       1\n",
       "       17753     20\n",
       "       16408      1\n",
       "       8217       4\n",
       "       4123       2\n",
       "       4124       1\n",
       "       31         1\n",
       "       12320      1\n",
       "       4129      11\n",
       "       16418      1\n",
       "       9563       1\n",
       "       12294      4\n",
       "       19121      2\n",
       "       20520      1\n",
       "       4139       2\n",
       "       16428      1\n",
       "       20526      1\n",
       "       12335      1\n",
       "       6152       9\n",
       "...              ..\n",
       "16     20414      1\n",
       "       4031       4\n",
       "       24512      2\n",
       "       5451       2\n",
       "       16324      1\n",
       "       4086       1\n",
       "       20424      1\n",
       "       24521      1\n",
       "       6954       1\n",
       "       20430      1\n",
       "       16638      1\n",
       "       4050       1\n",
       "       16339      3\n",
       "       16340      1\n",
       "       16343      1\n",
       "       8154       1\n",
       "       20446      1\n",
       "       12256      2\n",
       "       8161       1\n",
       "       16354      1\n",
       "       8165       1\n",
       "       12266      1\n",
       "       16363      1\n",
       "       4076       2\n",
       "       16365      1\n",
       "       12275      1\n",
       "       16374      1\n",
       "       12280      1\n",
       "       4089       9\n",
       "       16382      3\n",
       "\n",
       "[58745 rows x 1 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sparse_bow"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Convert to Doc-Topic Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'import pandas as pd\\n\\ndoc2id = {value : key for key, value in doc_ids.items()}\\ntype2id = {value : key for key, value in id_types.items()}\\n\\ncols = [doc2id[key] for key in set(sparse_bow.index.get_level_values(\"doc_id\"))]\\n#idx = [type2id[key] for key in set(sparse_bow.index.get_level_values(\"token_id\"))]\\n\\nset(sparse_bow.index.get_level_values(\"token_id\"))\\n\\n#doctopic_matrix = pd.DataFrame(columns=cols, index=idx)'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''import pandas as pd\n",
    "\n",
    "doc2id = {value : key for key, value in doc_ids.items()}\n",
    "type2id = {value : key for key, value in id_types.items()}\n",
    "\n",
    "cols = [doc2id[key] for key in set(sparse_bow.index.get_level_values(\"doc_id\"))]\n",
    "#idx = [type2id[key] for key in set(sparse_bow.index.get_level_values(\"token_id\"))]\n",
    "\n",
    "set(sparse_bow.index.get_level_values(\"token_id\"))\n",
    "\n",
    "#doctopic_matrix = pd.DataFrame(columns=cols, index=idx)'''"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Remove Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os.path\n",
    "basepath =  os.path.abspath(os.path.join(\".\", os.pardir))\n",
    "\n",
    "with open(os.path.join(basepath, \"Topics/tutorial_supplementals/stopwords/en\"), 'r', encoding = 'utf-8') as f: \n",
    "    stopword_list = f.read().split('\\n')\n",
    "    \n",
    "stopword_list = set(stopword_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sparse_df_stopwords_removed = pre.remove_features(sparse_bow, id_types, stopword_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "58745\n",
      "53040\n"
     ]
    }
   ],
   "source": [
    "print(len(sparse_bow))\n",
    "print(len(sparse_df_stopwords_removed))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Create Feature Remove Lists"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['the',\n",
       " 'of',\n",
       " 'and',\n",
       " 'to',\n",
       " 'a',\n",
       " 'i',\n",
       " 'in',\n",
       " 'he',\n",
       " 'that',\n",
       " 'was',\n",
       " 'it',\n",
       " 'his',\n",
       " 'is',\n",
       " 'as',\n",
       " 'with',\n",
       " 'had',\n",
       " 'you',\n",
       " 'we',\n",
       " 'at',\n",
       " 'for',\n",
       " 'have',\n",
       " 'but',\n",
       " 'which',\n",
       " 'my',\n",
       " 'not',\n",
       " 'on',\n",
       " 'said',\n",
       " 'be',\n",
       " 'from',\n",
       " 'him',\n",
       " 'all',\n",
       " 'this',\n",
       " 'there',\n",
       " 'were',\n",
       " 'me',\n",
       " 'by',\n",
       " 'they',\n",
       " 'no',\n",
       " 'one',\n",
       " 'her',\n",
       " 'so',\n",
       " 'are',\n",
       " 'or',\n",
       " 'an',\n",
       " 'our',\n",
       " 'been',\n",
       " 'out',\n",
       " 'up',\n",
       " 'she',\n",
       " 'would',\n",
       " 'upon',\n",
       " 'what',\n",
       " 'when',\n",
       " 'could',\n",
       " 'if',\n",
       " 'then',\n",
       " 'will',\n",
       " 'into',\n",
       " 'their',\n",
       " 'us',\n",
       " 'man',\n",
       " 'them',\n",
       " 'more',\n",
       " 'now',\n",
       " 'very',\n",
       " 'down',\n",
       " 'who',\n",
       " 'some',\n",
       " 'has',\n",
       " 'do',\n",
       " 'about',\n",
       " 'your',\n",
       " 'any',\n",
       " 'little',\n",
       " 'over',\n",
       " 'than',\n",
       " 'did',\n",
       " 'only',\n",
       " 'like',\n",
       " 'other',\n",
       " 'through',\n",
       " 'its',\n",
       " 'time',\n",
       " 'see',\n",
       " 'can',\n",
       " 'came',\n",
       " 'come',\n",
       " 'back',\n",
       " 'two',\n",
       " 'before',\n",
       " 'must',\n",
       " 'where',\n",
       " 'know',\n",
       " 'here',\n",
       " 'great',\n",
       " 'holmes',\n",
       " 'well',\n",
       " 'these',\n",
       " 'may',\n",
       " 'how']"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stopword_list = pre.find_stopwords(sparse_bow, id_types, 100)\n",
    "stopword_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "21187"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hapax_list = pre.find_hapax(sparse_bow, id_types)\n",
    "len(hapax_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Remove Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "feature_list = set(stopword_list).union(hapax_list)\n",
    "clean_term_frequency = pre.remove_features(sparse_bow, id_types, feature_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4874"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(clean_term_frequency)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Topic Modeling with Gensim"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Saving Sparse BOW"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "num_docs = max(sparse_bow.index.get_level_values(\"doc_id\"))\n",
    "num_types = max(sparse_bow.index.get_level_values(\"token_id\"))\n",
    "sum_counts = sum(sparse_bow[0])\n",
    "\n",
    "header_string = str(num_docs) + \" \" + str(num_types) + \" \" + str(sum_counts) + \"\\n\"\n",
    "\n",
    "with open(\"gb_plain.mm\", 'a', encoding = \"utf-8\") as f:\n",
    "    f.write(\"%%MatrixMarket matrix coordinate real general\\n\")\n",
    "    f.write(header_string)\n",
    "    sparse_bow.to_csv( f, sep = ' ', header = None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from gensim.models import LdaModel\n",
    "from gensim.corpora import MmCorpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mm = MmCorpus(\"gb_plain.mm\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#in case you're only loading the corpus - build dict first\n",
    "\n",
    "#import pickle\n",
    "\n",
    "#pickle.dump( id_types, open( \"gb_plain.dictionary\", \"wb\" ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#id_types = pickle.load(open(\"gb_plain.dictionary\", 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#mm = gensim.corpora.MmCorpus(\"gb_all.mm\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#id_types = pickle.load(open(\"gb_all.dictionary\", 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "doc2id = {value : key for key, value in doc_ids.items()}\n",
    "type2id = {value : key for key, value in id_types.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#model = LdaModel(corpus=mm, id2word=type2id, num_topics=60, alpha = \"symmetric\", passes = 10) #import momentan in visual \n",
    "# -> da ich mir noch nicht sicher bin, welche Funktionen in das tm_gensim.py sollen\n",
    "model = LdaModel(corpus=mm, id2word=type2id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['straightline',\n",
       " 'geographer',\n",
       " 'wrinkles',\n",
       " 'fact_’—but',\n",
       " 'melville',\n",
       " 'dismissing',\n",
       " 'inessentiality',\n",
       " 'steeply',\n",
       " 'furnish',\n",
       " 'landau']"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "topic_nr_x = model.get_topic_terms(10)\n",
    "\n",
    "[type2id[i[0]] for i in topic_nr_x]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "topics = model.show_topics(num_topics = 60)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(71,\n",
       "  '0.000*\"watch\" + 0.000*\"blighter\" + 0.000*\"obadiah\" + 0.000*\"gangs\" + 0.000*\"settlers\" + 0.000*\"whenever\" + 0.000*\"incidentally\" + 0.000*\"astern\" + 0.000*\"sidled\" + 0.000*\"awaked\"'),\n",
       " (32,\n",
       "  '0.000*\"outré\" + 0.000*\"astronomy\" + 0.000*\"mogul\" + 0.000*\"trampling\" + 0.000*\"chimed\" + 0.000*\"eh\" + 0.000*\"regions\" + 0.000*\"gnaws\" + 0.000*\"slave\\'s\" + 0.000*\"master-an\\'-missus\"'),\n",
       " (78,\n",
       "  '0.000*\"gold-braided\" + 0.000*\"crowd\" + 0.000*\"dauntless\" + 0.000*\"mailed\" + 0.000*\"out--thus\" + 0.000*\"goaded\" + 0.000*\"daze\" + 0.000*\"bones--that\" + 0.000*\"discerned\" + 0.000*\"desk\"'),\n",
       " (45,\n",
       "  '0.000*\"cod\" + 0.000*\"numerousness\" + 0.000*\"snatching\" + 0.000*\"clean-cut\" + 0.000*\"upper-story\" + 0.000*\"comparisons\" + 0.000*\"telegraphed\" + 0.000*\"dash-all\" + 0.000*\"aided\" + 0.000*\"yard\"'),\n",
       " (6,\n",
       "  '0.000*\"rebirth\" + 0.000*\"bragi\\'s\" + 0.000*\"fool\\'s\" + 0.000*\"accentuate\" + 0.000*\"forget\" + 0.000*\"carved\" + 0.000*\"smothered\" + 0.000*\"herb\" + 0.000*\"come-down\" + 0.000*\"naturæ\"'),\n",
       " (75,\n",
       "  '0.000*\"this\" + 0.000*\"side-streets\" + 0.000*\"sheared\" + 0.000*\"awake\" + 0.000*\"saucer\" + 0.000*\"entrusted\" + 0.000*\"stratum’s\" + 0.000*\"example:—admitting\" + 0.000*\"jacques\" + 0.000*\"reavers\"'),\n",
       " (93,\n",
       "  '0.000*\"congratulating\" + 0.000*\"slap\" + 0.000*\"pitched\" + 0.000*\"ochen\" + 0.000*\"supine\" + 0.000*\"gentes\\'--published\" + 0.000*\"pools\" + 0.000*\"badger-colored\" + 0.000*\"heaved\" + 0.000*\"masculine\"'),\n",
       " (87,\n",
       "  '0.000*\"saurians\" + 0.000*\"abnormal—of\" + 0.000*\"fish-head\" + 0.000*\"implicates\" + 0.000*\"science—from\" + 0.000*\"diminish\" + 0.000*\"burly\" + 0.000*\"participant\" + 0.000*\"fro\" + 0.000*\"sole\"'),\n",
       " (56,\n",
       "  '0.000*\"burgomaster\" + 0.000*\"ducks\" + 0.000*\"breath\" + 0.000*\"umbrage-his-ma\" + 0.000*\"hhrrmph\" + 0.000*\"relative\" + 0.000*\"contiguous\" + 0.000*\"well-set\" + 0.000*\"pull-up\" + 0.000*\"indignant\"'),\n",
       " (28,\n",
       "  '0.000*\"eligible\" + 0.000*\"landing\" + 0.000*\"street-arabs\" + 0.000*\"desk\" + 0.000*\"eyrie\" + 0.000*\"beach--old\" + 0.000*\"spitting\" + 0.000*\"day\" + 0.000*\"carefully\" + 0.000*\"cognizance—at\"'),\n",
       " (67,\n",
       "  '0.000*\"stethoscope\" + 0.000*\"chart\" + 0.000*\"fellow-men\" + 0.000*\"calf-shed\" + 0.000*\"boundless\" + 0.000*\"moons\" + 0.000*\"actual\" + 0.000*\"assassins\" + 0.000*\"story\" + 0.000*\"last-run-of-season\"'),\n",
       " (24,\n",
       "  '0.000*\"overcoming\" + 0.000*\"confronted\" + 0.000*\"back--and\" + 0.000*\"hornet\\'s\" + 0.000*\"thus\" + 0.000*\"mister-kent-people\" + 0.000*\"tighted\" + 0.000*\"wrote\" + 0.000*\"particularly\" + 0.000*\"starfish\"'),\n",
       " (85,\n",
       "  '0.000*\"breakdown\" + 0.000*\"outwitted\" + 0.000*\"rehoboth\" + 0.000*\"dreamily\" + 0.000*\"parlors\" + 0.000*\"hundredths\" + 0.000*\"redness\" + 0.000*\"admitted\" + 0.000*\"ahoo\" + 0.000*\"row\"'),\n",
       " (35,\n",
       "  '0.000*\"savage\" + 0.000*\"halcyon\" + 0.000*\"dancer\" + 0.000*\"happy\" + 0.000*\"bronze\" + 0.000*\"heel\" + 0.000*\"notepaper\" + 0.000*\"house-physician--little\" + 0.000*\"dagonia\" + 0.000*\"starboard\"'),\n",
       " (18,\n",
       "  '0.000*\"bode—be\" + 0.000*\"qu’il\" + 0.000*\"condescend\" + 0.000*\"superseded\" + 0.000*\"jabber\" + 0.000*\"possessions--guns\" + 0.000*\"interlude\" + 0.000*\"granting\" + 0.000*\"closed\" + 0.000*\"horns\"'),\n",
       " (42,\n",
       "  '0.000*\"odor—came\" + 0.000*\"moisture\" + 0.000*\"red-eyed\" + 0.000*\"slowly\" + 0.000*\"vegetation—of\" + 0.000*\"all\" + 0.000*\"saluted\" + 0.000*\"sacred\" + 0.000*\"watson\" + 0.000*\"battering-ram\"'),\n",
       " (20,\n",
       "  '0.000*\"eyeballs\" + 0.000*\"sublime—some\" + 0.000*\"solidity—of\" + 0.000*\"mind\" + 0.000*\"stove\" + 0.000*\"bellies\" + 0.000*\"conolly\\'s\" + 0.000*\"harvest\" + 0.000*\"grotesque\" + 0.000*\"richness\"'),\n",
       " (4,\n",
       "  '0.000*\"bother\" + 0.000*\"denudation\" + 0.000*\"rack\" + 0.000*\"brilliantly\" + 0.000*\"akbitana\" + 0.000*\"pardon\" + 0.000*\"try\" + 0.000*\"perpetual—the\" + 0.000*\"cognizance—at\" + 0.000*\"billion—leaving\"'),\n",
       " (99,\n",
       "  '0.000*\"indignity\" + 0.000*\"crescent\" + 0.000*\"rescuing\" + 0.000*\"sticky\" + 0.000*\"prisoner\" + 0.000*\"chased\" + 0.000*\"seamed\" + 0.000*\"availability\" + 0.000*\"brain-work\" + 0.000*\"truffle\"'),\n",
       " (43,\n",
       "  '0.000*\"cotswold\" + 0.000*\"embodied\" + 0.000*\"blacker\" + 0.000*\"electronic\" + 0.000*\"curved\" + 0.000*\"zingara\" + 0.000*\"cosmos\" + 0.000*\"love-taps\" + 0.000*\"logical\" + 0.000*\"lifting\"'),\n",
       " (30,\n",
       "  '0.000*\"claims\" + 0.000*\"yarders\" + 0.000*\"stoicism\" + 0.000*\"shoutings\" + 0.000*\"samoa\" + 0.000*\"appa\" + 0.000*\"winds—mentioned\" + 0.000*\"pershore\" + 0.000*\"weirdest\" + 0.000*\"northwestward\"'),\n",
       " (17,\n",
       "  '0.000*\"grass\" + 0.000*\"stammering\" + 0.000*\"fixture\" + 0.000*\"gods--very\" + 0.000*\"nineteenth\" + 0.000*\"usurper-blood\" + 0.000*\"plan—which\" + 0.000*\"inferring\" + 0.000*\"down-stream\" + 0.000*\"trot\"'),\n",
       " (40,\n",
       "  '0.000*\"partisan\" + 0.000*\"moon-attended\" + 0.000*\"ornamented\" + 0.000*\"tube\" + 0.000*\"applied\" + 0.000*\"sighed\" + 0.000*\"defiance\" + 0.000*\"minutiae\" + 0.000*\"leading\" + 0.000*\"accentuated\"'),\n",
       " (1,\n",
       "  '0.000*\"males\" + 0.000*\"agitation\" + 0.000*\"lynx\" + 0.000*\"shemites\" + 0.000*\"deal\" + 0.000*\"barrels\" + 0.000*\"error—lay\" + 0.000*\"cursory\" + 0.000*\"ear-shot\" + 0.000*\"electro-chemical\"'),\n",
       " (11,\n",
       "  '0.000*\"picketed\" + 0.000*\"rëaction\" + 0.000*\"blackmailing\" + 0.000*\"atom_—here\" + 0.000*\"ferrier\" + 0.000*\"holmes--his\" + 0.000*\"father\\'s\" + 0.000*\"currency\" + 0.000*\"school-ruler\" + 0.000*\"ensure\"'),\n",
       " (8,\n",
       "  '0.000*\"brainless\" + 0.000*\"sulkily\" + 0.000*\"coffers\" + 0.000*\"monomania\" + 0.000*\"bee\" + 0.000*\"tiger\" + 0.000*\"tunnel\" + 0.000*\"astronomy\" + 0.000*\"banished\" + 0.000*\"dictated\"'),\n",
       " (97,\n",
       "  '0.000*\"eye-sight\" + 0.000*\"whirl\" + 0.000*\"fit\" + 0.000*\"category\" + 0.000*\"marsh\" + 0.000*\"faultless\" + 0.000*\"docks\" + 0.000*\"willingly\" + 0.000*\"interminable\" + 0.000*\"going\"'),\n",
       " (58,\n",
       "  '0.000*\"significant\" + 0.000*\"nightingale\" + 0.000*\"full-of\" + 0.000*\"pink-tinted\" + 0.000*\"nasal\" + 0.000*\"centripetal\" + 0.000*\"suk\" + 0.000*\"secreted\" + 0.000*\"complex:—it\" + 0.000*\"deeps\"'),\n",
       " (60,\n",
       "  '0.000*\"panther\\'s\" + 0.000*\"ristori\" + 0.000*\"moon:—three\" + 0.000*\"prodded\" + 0.000*\"hurried\" + 0.000*\"figure\" + 0.000*\"reversal\" + 0.000*\"rescuing\" + 0.000*\"virulent\" + 0.000*\"pickets--\"here\"'),\n",
       " (33,\n",
       "  '0.000*\"permanently\" + 0.000*\"enlargements\" + 0.000*\"possible\" + 0.000*\"posteriori\" + 0.000*\"halle\\'s\" + 0.000*\"conclusions\" + 0.000*\"up--to\" + 0.000*\"revelation\" + 0.000*\"infrequently\" + 0.000*\"hosts\"'),\n",
       " (46,\n",
       "  '0.000*\"descriptive\" + 0.000*\"cawnpore\" + 0.000*\"lovies\" + 0.000*\"timbersides\\'--this\" + 0.000*\"defect\" + 0.000*\"symbolism\" + 0.000*\"country-journal\" + 0.000*\"rat-holes\" + 0.000*\"clumps\" + 0.000*\"chiseling\"'),\n",
       " (91,\n",
       "  '0.000*\"contained\" + 0.000*\"overcame\" + 0.000*\"cubical\" + 0.000*\"almighty\" + 0.000*\"diffusion—are\" + 0.000*\"connaissons\" + 0.000*\"arrogant\" + 0.000*\"doings\" + 0.000*\"tarpaulins\" + 0.000*\"vanishing\"'),\n",
       " (31,\n",
       "  '0.000*\"tales\" + 0.000*\"sets\" + 0.000*\"smartness\" + 0.000*\"badly\" + 0.000*\"flashlights\" + 0.000*\"deceptive\" + 0.000*\"taciturn\" + 0.000*\"kennels-that\" + 0.000*\"barked\" + 0.000*\"bye-loed\"'),\n",
       " (37,\n",
       "  '0.000*\"striding\" + 0.000*\"palmyra\" + 0.000*\"size_—of\" + 0.000*\"expressions\" + 0.000*\"generalization\" + 0.000*\"settlers\" + 0.000*\"scrutinized\" + 0.000*\"sub_-principle\" + 0.000*\"revolve\" + 0.000*\"messed\"'),\n",
       " (65,\n",
       "  '0.000*\"gesticulated\" + 0.000*\"nature—a\" + 0.000*\"un\" + 0.000*\"perplexing\" + 0.000*\"true--it\" + 0.000*\"shrine--once\" + 0.000*\"maxillary\" + 0.000*\"sarcasms\" + 0.000*\"tiptoe\" + 0.000*\"routes_—but\"'),\n",
       " (74,\n",
       "  '0.000*\"ferocious\" + 0.000*\"also,—a\" + 0.000*\"extricated\" + 0.000*\"get-away-out-of\" + 0.000*\"more\" + 0.000*\"advising\" + 0.000*\"abbey\" + 0.000*\"proceed—until\" + 0.000*\"babblings\" + 0.000*\"quantity_—that\"'),\n",
       " (25,\n",
       "  '0.000*\"tutor\" + 0.000*\"boring\" + 0.000*\"horrible\" + 0.000*\"red-letter\" + 0.000*\"succeeded\" + 0.000*\"border\" + 0.000*\"reversion\" + 0.000*\"bachelor\\'s\" + 0.000*\"dashed\" + 0.000*\"he\\'ll\"'),\n",
       " (81,\n",
       "  '0.000*\"tremulousness\" + 0.000*\"gout\" + 0.000*\"address\" + 0.000*\"tries\" + 0.000*\"influence_—solely\" + 0.000*\"overtaken\" + 0.000*\"horn-on-a-fine-hunting-morn\" + 0.000*\"day--thursday\" + 0.000*\"whalers\" + 0.000*\"cup\"'),\n",
       " (70,\n",
       "  '0.000*\"clarendon\" + 0.000*\"erroneous\" + 0.000*\"crossed\" + 0.000*\"form\" + 0.000*\"heavily\" + 0.000*\"containing\" + 0.000*\"light-impression—or\" + 0.000*\"painful\" + 0.000*\"audley\" + 0.000*\"accounted\"'),\n",
       " (21,\n",
       "  '0.000*\"anew\" + 0.000*\"arithmetic\" + 0.000*\"example:—admitting\" + 0.000*\"brochure\" + 0.000*\"foulness\" + 0.000*\"foe\\'s\" + 0.000*\"inferences—have\" + 0.000*\"smallest\\'s\" + 0.000*\"nightfall\" + 0.000*\"open-minded\"'),\n",
       " (9,\n",
       "  '0.000*\"cordage\" + 0.000*\"foggy\" + 0.000*\"curling\" + 0.000*\"black\\'s\" + 0.000*\"essentiality\" + 0.000*\"heterogeneity\" + 0.000*\"stump\" + 0.000*\"wild-beast\" + 0.000*\"miscreants\" + 0.000*\"who\"'),\n",
       " (55,\n",
       "  '0.000*\"shapely\" + 0.000*\"trunks\" + 0.000*\"pop-gun\" + 0.000*\"cut-throat\" + 0.000*\"distilled\" + 0.000*\"troops\" + 0.000*\"nights\" + 0.000*\"degenerate\" + 0.000*\"equivocal\" + 0.000*\"fostering\"'),\n",
       " (41,\n",
       "  '0.000*\"strains\" + 0.000*\"growths\" + 0.000*\"unbosomed\" + 0.000*\"commenced\" + 0.000*\"piano\" + 0.000*\"boat-shed\" + 0.000*\"gasoline\" + 0.000*\"disgustingly\" + 0.000*\"girdles\" + 0.000*\"brother\"'),\n",
       " (50,\n",
       "  '0.000*\"paper\" + 0.000*\"placing\" + 0.000*\"yellow-haired\" + 0.000*\"steerage-way\" + 0.000*\"cygni\" + 0.000*\"idly\" + 0.000*\"bang-stick\" + 0.000*\"devices\" + 0.000*\"petrification\" + 0.000*\"outlook\"'),\n",
       " (64,\n",
       "  '0.000*\"piquancy\" + 0.000*\"microscopes\" + 0.000*\"religious\" + 0.000*\"amends\" + 0.000*\"groom\" + 0.000*\"slightly\" + 0.000*\"they\\'ve\" + 0.000*\"vanishes\" + 0.000*\"fluent\" + 0.000*\"hatred\"'),\n",
       " (39,\n",
       "  '0.000*\"incline\" + 0.000*\"kaa\\'s\" + 0.000*\"connoisseurship\" + 0.000*\"beetles\" + 0.000*\"nowadays\" + 0.000*\"patiently\" + 0.000*\"camest\" + 0.000*\"sandals\" + 0.000*\"fwom\" + 0.000*\"species\"'),\n",
       " (94,\n",
       "  '0.000*\"puritan\" + 0.000*\"cantered\" + 0.000*\"need\" + 0.000*\"stalk\" + 0.000*\"calls\" + 0.000*\"dresser\" + 0.000*\"duplicated\" + 0.000*\"tended\" + 0.000*\"horn\" + 0.000*\"heterogeneous\"'),\n",
       " (16,\n",
       "  '0.000*\"school-ruler\" + 0.000*\"diffusion—as\" + 0.000*\"rama\\'s\" + 0.000*\"tone\" + 0.000*\"fantastic\" + 0.000*\"brigade\" + 0.000*\"fastness\" + 0.000*\"nathaniel\" + 0.000*\"wounds\" + 0.000*\"narrate\"'),\n",
       " (79,\n",
       "  '0.000*\"teach\" + 0.000*\"industry\" + 0.000*\"worlds—or\" + 0.000*\"cattle-herding\" + 0.000*\"parry\" + 0.000*\"asserting\" + 0.000*\"nineteen--with\" + 0.000*\"spend-happy-evening\" + 0.000*\"soured\" + 0.000*\"police-station\"'),\n",
       " (62,\n",
       "  '0.000*\"harborers\" + 0.000*\"rebuke\" + 0.000*\"bowl\" + 0.000*\"caress\" + 0.000*\"bidding\" + 0.000*\"fancied\" + 0.000*\"indeterminate\" + 0.000*\"relied\" + 0.000*\"cities\" + 0.000*\"prevents\"'),\n",
       " (57,\n",
       "  '0.000*\"umbrage-his-ma\" + 0.000*\"this—a\" + 0.000*\"gate\" + 0.000*\"indisputable\" + 0.000*\"sharply\" + 0.000*\"euphonious\" + 0.000*\"idiosyncrasy\" + 0.000*\"escapes\" + 0.000*\"clerks\" + 0.000*\"peopled\"'),\n",
       " (13,\n",
       "  '0.000*\"daintiness\" + 0.000*\"spiteful\" + 0.000*\"stained\" + 0.000*\"cognizance—at\" + 0.000*\"years\" + 0.000*\"sickness\" + 0.000*\"man-cub\" + 0.000*\"balustraded\" + 0.000*\"perhaps--and\" + 0.000*\"michaelmas\"'),\n",
       " (80,\n",
       "  '0.000*\"fields\" + 0.000*\"flag-wagging\" + 0.000*\"renamed\" + 0.000*\"nondescripts\" + 0.000*\"crashing\" + 0.000*\"poet\" + 0.000*\"disputing\" + 0.000*\"litigation\" + 0.000*\"naughtiness\" + 0.000*\"roar\"'),\n",
       " (92,\n",
       "  '0.000*\"public\" + 0.000*\"make\" + 0.000*\"vigilant\" + 0.000*\"bears\" + 0.000*\"disordered\" + 0.000*\"goose\" + 0.000*\"stopping\" + 0.000*\"irrelation\" + 0.000*\"cheek\" + 0.000*\"baal-pteor\\'s\"'),\n",
       " (22,\n",
       "  '0.000*\"downy\" + 0.000*\"moment—in\" + 0.000*\"repetitions\" + 0.000*\"visibility\" + 0.000*\"ripple\" + 0.000*\"showery\" + 0.000*\"palaeotheres\" + 0.000*\"channels\" + 0.000*\"lecoq\" + 0.000*\"predominates\"'),\n",
       " (19,\n",
       "  '0.000*\"jammed\" + 0.000*\"profession\" + 0.000*\"dominions\" + 0.000*\"mickey\" + 0.000*\"screw\" + 0.000*\"animal\\'s\" + 0.000*\"carved\" + 0.000*\"bonnet\" + 0.000*\"disguise\" + 0.000*\"kin\"'),\n",
       " (73,\n",
       "  '0.000*\"scaffold\" + 0.000*\"whiskey-pegs\" + 0.000*\"parti-striped\" + 0.000*\"well-appointed\" + 0.000*\"limitless\" + 0.000*\"sterneninseln\" + 0.000*\"feather\" + 0.000*\"both\" + 0.000*\"nag\\'s\" + 0.000*\"reliable\"'),\n",
       " (98,\n",
       "  '0.000*\"reformat\" + 0.000*\"riposted\" + 0.000*\"rocks--the\" + 0.000*\"outcries\" + 0.000*\"captive\\'s\" + 0.000*\"outranks\" + 0.000*\"forgetting\" + 0.000*\"journeyed\" + 0.000*\"musket\" + 0.000*\"out-heroded\"'),\n",
       " (54,\n",
       "  '0.000*\"rovers\" + 0.000*\"utter\" + 0.000*\"myself—“unity\" + 0.000*\"patience\" + 0.000*\"dominate\" + 0.000*\"reeving\" + 0.000*\"rifles\" + 0.000*\"shed\" + 0.000*\"influence_—solely\" + 0.000*\"lessening\"'),\n",
       " (84,\n",
       "  '0.000*\"upward\" + 0.000*\"gent\" + 0.000*\"put-in\" + 0.000*\"lawnmower\" + 0.000*\"truffles\" + 0.000*\"per\" + 0.000*\"investigativeness\" + 0.000*\"ram\" + 0.000*\"frantic\" + 0.000*\"molds\"')]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "topics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Topic Modeling mit Mallet"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Mallet Binary erzeugen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'C:\\\\Users\\\\Philip\\\\Documents\\\\Python35-32_Scripts\\\\Funs\\\\DARIAH'"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os.path\n",
    "basepath =  os.path.abspath(os.path.join(\".\", os.pardir))\n",
    "basepath"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['mallet', 'import-dir', '--input', 'C:\\\\Users\\\\Philip\\\\Documents\\\\Python35-32_Scripts\\\\Funs\\\\DARIAH\\\\Topics\\\\corpus_txt', '--output', 'mallet_output\\\\malletBinary.mallet', '--keep-sequence', '--remove-stopwords']\n"
     ]
    }
   ],
   "source": [
    "#path_to_mallet = \"/home/sina/Uni/Dariah/mallet/bin/mallet\"\n",
    "#path_to_mallet = \"mallet\"\n",
    "path_to_corpus = os.path.join(basepath, \"Topics\\\\corpus_txt\")\n",
    "\n",
    "mal.create_mallet_binary(path_to_corpus)\n",
    "\n",
    "#toDo absolute Pfade"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Mallet output erzeugen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "outfolder = \"mallet_output\" \n",
    "path_to_mallet = \"mallet\"\n",
    "malletBinary = \"mallet_output\\\\malletBinary.mallet\"\n",
    "mal.create_mallet_model(malletBinary, outfolder)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Visualisierung "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"lda_model = 'out_easy/corpus.lda'\\ncorpus = 'out_easy/corpus.mm'\\ndictionary = 'out_easy/corpus.dict'\\ndoc_labels = 'out_easy/corpus_doclabels.txt'\\ninteractive  = False\\n\\nvis = visual.Visualization(lda_model, corpus, dictionary, doc_labels, interactive)\""
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''lda_model = 'out_easy/corpus.lda'\n",
    "corpus = 'out_easy/corpus.mm'\n",
    "dictionary = 'out_easy/corpus.dict'\n",
    "doc_labels = 'out_easy/corpus_doclabels.txt'\n",
    "interactive  = False\n",
    "\n",
    "vis = visual.Visualization(lda_model, corpus, dictionary, doc_labels, interactive)'''"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Heatmap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#heatmap = visual.make_heatmap()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#visual.save_heatmap(\"./visualizations/heatmap\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Interactive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#vis = collection.Visualization(lda_model, corpus, dictionary, doc_labels, interactive=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#vis.make_interactive()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#vis.save_interactive(\"./visualizations/interactive\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Ob ihr wirklich richtig steht, seht ihr, wenn ..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "![success](http://cdn2.hubspot.net/hub/128506/file-446943132-jpg/images/computer_woman_success.jpg)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}