IntegrationTest_v01.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\philip\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages\\funcy\\decorators.py:56: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead\n",
      "  spec = inspect.getargspec(func)\n",
      "c:\\users\\philip\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages\\funcy\\decorators.py:56: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead\n",
      "  spec = inspect.getargspec(func)\n",
      "c:\\users\\philip\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages\\funcy\\decorators.py:56: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead\n",
      "  spec = inspect.getargspec(func)\n",
      "c:\\users\\philip\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages\\funcy\\decorators.py:56: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead\n",
      "  spec = inspect.getargspec(func)\n",
      "c:\\users\\philip\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages\\funcy\\decorators.py:56: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead\n",
      "  spec = inspect.getargspec(func)\n",
      "c:\\users\\philip\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages\\funcy\\decorators.py:56: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead\n",
      "  spec = inspect.getargspec(func)\n"
     ]
    }
   ],
   "source": [
    "from dariah_topics import preprocessing as pre\n",
    "from dariah_topics import visualization as visual\n",
    "from dariah_topics import mallet as mal\n",
    "# Warning is Gensim related"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Liste mit Dateinamen erzeugen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\philip\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages\\IPython\\core\\formatters.py:92: DeprecationWarning: DisplayFormatter._ipython_display_formatter_default is deprecated: use @default decorator instead.\n",
      "  def _ipython_display_formatter_default(self):\n",
      "c:\\users\\philip\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages\\IPython\\core\\formatters.py:98: DeprecationWarning: DisplayFormatter._formatters_default is deprecated: use @default decorator instead.\n",
      "  def _formatters_default(self):\n",
      "c:\\users\\philip\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages\\IPython\\core\\formatters.py:677: DeprecationWarning: PlainTextFormatter._deferred_printers_default is deprecated: use @default decorator instead.\n",
      "  def _deferred_printers_default(self):\n",
      "c:\\users\\philip\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages\\IPython\\core\\formatters.py:669: DeprecationWarning: PlainTextFormatter._singleton_printers_default is deprecated: use @default decorator instead.\n",
      "  def _singleton_printers_default(self):\n",
      "c:\\users\\philip\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages\\IPython\\core\\formatters.py:672: DeprecationWarning: PlainTextFormatter._type_printers_default is deprecated: use @default decorator instead.\n",
      "  def _type_printers_default(self):\n",
      "c:\\users\\philip\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages\\IPython\\core\\formatters.py:669: DeprecationWarning: PlainTextFormatter._singleton_printers_default is deprecated: use @default decorator instead.\n",
      "  def _singleton_printers_default(self):\n",
      "c:\\users\\philip\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages\\IPython\\core\\formatters.py:672: DeprecationWarning: PlainTextFormatter._type_printers_default is deprecated: use @default decorator instead.\n",
      "  def _type_printers_default(self):\n",
      "c:\\users\\philip\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages\\IPython\\core\\formatters.py:677: DeprecationWarning: PlainTextFormatter._deferred_printers_default is deprecated: use @default decorator instead.\n",
      "  def _deferred_printers_default(self):\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['corpus_txt\\\\Doyle_AScandalinBohemia.txt',\n",
       " 'corpus_txt\\\\Doyle_AStudyinScarlet.txt',\n",
       " 'corpus_txt\\\\Doyle_TheHoundoftheBaskervilles.txt',\n",
       " 'corpus_txt\\\\Doyle_TheSignoftheFour.txt',\n",
       " 'corpus_txt\\\\Howard_GodsoftheNorth.txt']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path_txt = \"corpus_txt\"\n",
    "#path_txt = \"grenzbote_plain/*/\"\n",
    "\n",
    "doclist_txt = pre.create_document_list(path_txt)\n",
    "doclist_txt[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['corpus_csv\\\\Doyle_AStudyinScarlet.txt.csv',\n",
       " 'corpus_csv\\\\Doyle_TheHoundoftheBaskervilles.txt.csv',\n",
       " 'corpus_csv\\\\Doyle_TheSignoftheFour.txt.csv',\n",
       " 'corpus_csv\\\\Howard_GodsoftheNorth.txt.csv',\n",
       " 'corpus_csv\\\\Howard_SchadowsinZamboula.txt.csv']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path_csv = \"corpus_csv\"\n",
    "\n",
    "doclist_csv = pre.create_document_list(path_csv, 'csv')\n",
    "doclist_csv[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#####  Liste mit Dokumentenlabels erzeugen - (Funktion wird durch Thorsten's generischere Funktion ersetzt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['corpus_txt\\\\Doyle_AScandalinBohemia.txt',\n",
       " 'corpus_txt\\\\Doyle_AStudyinScarlet.txt',\n",
       " 'corpus_txt\\\\Doyle_TheHoundoftheBaskervilles.txt',\n",
       " 'corpus_txt\\\\Doyle_TheSignoftheFour.txt',\n",
       " 'corpus_txt\\\\Howard_GodsoftheNorth.txt']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doc_labels = list(pre.get_labels(doclist_txt))\n",
    "doc_labels[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Corpus laden"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "corpus_txt = pre.read_from_txt(doclist_txt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "corpus_csv = pre.read_from_csv(doclist_csv)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Tokenisieren"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['a',\n",
       "  'scandal',\n",
       "  'in',\n",
       "  'bohemia',\n",
       "  'a',\n",
       "  'conan',\n",
       "  'doyle',\n",
       "  'i',\n",
       "  'to',\n",
       "  'sherlock',\n",
       "  'holmes',\n",
       "  'she',\n",
       "  'is',\n",
       "  'always',\n",
       "  'the',\n",
       "  'woman',\n",
       "  'i',\n",
       "  'have',\n",
       "  'seldom',\n",
       "  'heard',\n",
       "  'him',\n",
       "  'mention',\n",
       "  'her',\n",
       "  'under',\n",
       "  'any',\n",
       "  'other',\n",
       "  'name',\n",
       "  'in',\n",
       "  'his',\n",
       "  'eyes',\n",
       "  'she',\n",
       "  'eclipses',\n",
       "  'and',\n",
       "  'predominates',\n",
       "  'the',\n",
       "  'whole',\n",
       "  'of',\n",
       "  'her',\n",
       "  'sex',\n",
       "  'it',\n",
       "  'was',\n",
       "  'not',\n",
       "  'that',\n",
       "  'he',\n",
       "  'felt',\n",
       "  'any',\n",
       "  'emotion',\n",
       "  'akin',\n",
       "  'to',\n",
       "  'love',\n",
       "  'for',\n",
       "  'irene',\n",
       "  'adler',\n",
       "  'all',\n",
       "  'emotions',\n",
       "  'and',\n",
       "  'that',\n",
       "  'one',\n",
       "  'particularly',\n",
       "  'were',\n",
       "  'abhorrent',\n",
       "  'to',\n",
       "  'his',\n",
       "  'cold',\n",
       "  'precise',\n",
       "  'but',\n",
       "  'admirably',\n",
       "  'balanced',\n",
       "  'mind',\n",
       "  'he',\n",
       "  'was',\n",
       "  'i',\n",
       "  'take',\n",
       "  'it',\n",
       "  'the',\n",
       "  'most',\n",
       "  'perfect',\n",
       "  'reasoning',\n",
       "  'and',\n",
       "  'observing',\n",
       "  'machine',\n",
       "  'that',\n",
       "  'the',\n",
       "  'world',\n",
       "  'has',\n",
       "  'seen',\n",
       "  'but',\n",
       "  'as',\n",
       "  'a',\n",
       "  'lover',\n",
       "  'he',\n",
       "  'would',\n",
       "  'have',\n",
       "  'placed',\n",
       "  'himself',\n",
       "  'in',\n",
       "  'a',\n",
       "  'false',\n",
       "  'position',\n",
       "  'he',\n",
       "  'never',\n",
       "  'spoke',\n",
       "  'of',\n",
       "  'the',\n",
       "  'softer',\n",
       "  'passions',\n",
       "  'save',\n",
       "  'with',\n",
       "  'a',\n",
       "  'gibe',\n",
       "  'and',\n",
       "  'a',\n",
       "  'sneer',\n",
       "  'they',\n",
       "  'were',\n",
       "  'admirable',\n",
       "  'things',\n",
       "  'for',\n",
       "  'the',\n",
       "  'observer--excellent',\n",
       "  'for',\n",
       "  'drawing',\n",
       "  'the',\n",
       "  'veil',\n",
       "  'from',\n",
       "  \"men's\",\n",
       "  'motives',\n",
       "  'and',\n",
       "  'actions',\n",
       "  'but',\n",
       "  'for',\n",
       "  'the',\n",
       "  'trained',\n",
       "  'reasoner',\n",
       "  'to',\n",
       "  'admit',\n",
       "  'such',\n",
       "  'intrusions',\n",
       "  'into',\n",
       "  'his',\n",
       "  'own',\n",
       "  'delicate',\n",
       "  'and',\n",
       "  'finely',\n",
       "  'adjusted',\n",
       "  'temperament',\n",
       "  'was',\n",
       "  'to',\n",
       "  'introduce',\n",
       "  'a',\n",
       "  'distracting',\n",
       "  'factor',\n",
       "  'which',\n",
       "  'might',\n",
       "  'throw',\n",
       "  'a',\n",
       "  'doubt',\n",
       "  'upon',\n",
       "  'all',\n",
       "  'his',\n",
       "  'mental',\n",
       "  'results',\n",
       "  'grit',\n",
       "  'in',\n",
       "  'a',\n",
       "  'sensitive',\n",
       "  'instrument',\n",
       "  'or',\n",
       "  'a',\n",
       "  'crack',\n",
       "  'in',\n",
       "  'one',\n",
       "  'of',\n",
       "  'his',\n",
       "  'own',\n",
       "  'high-power',\n",
       "  'lenses',\n",
       "  'would',\n",
       "  'not',\n",
       "  'be',\n",
       "  'more',\n",
       "  'disturbing',\n",
       "  'than',\n",
       "  'a',\n",
       "  'strong',\n",
       "  'emotion',\n",
       "  'in',\n",
       "  'a',\n",
       "  'nature',\n",
       "  'such',\n",
       "  'as',\n",
       "  'his',\n",
       "  'and',\n",
       "  'yet',\n",
       "  'there',\n",
       "  'was',\n",
       "  'but',\n",
       "  'one',\n",
       "  'woman',\n",
       "  'to',\n",
       "  'him',\n",
       "  'and',\n",
       "  'that',\n",
       "  'woman',\n",
       "  'was',\n",
       "  'the',\n",
       "  'late',\n",
       "  'irene',\n",
       "  'adler',\n",
       "  'of',\n",
       "  'dubious',\n",
       "  'and',\n",
       "  'questionable',\n",
       "  'memory',\n",
       "  'i',\n",
       "  'had',\n",
       "  'seen',\n",
       "  'little',\n",
       "  'of',\n",
       "  'holmes',\n",
       "  'lately',\n",
       "  'my',\n",
       "  'marriage',\n",
       "  'had',\n",
       "  'drifted',\n",
       "  'us',\n",
       "  'away',\n",
       "  'from',\n",
       "  'each',\n",
       "  'other',\n",
       "  'my',\n",
       "  'own',\n",
       "  'complete',\n",
       "  'happiness',\n",
       "  'and',\n",
       "  'the',\n",
       "  'home-centred',\n",
       "  'interests',\n",
       "  'which',\n",
       "  'rise',\n",
       "  'up',\n",
       "  'around',\n",
       "  'the',\n",
       "  'man',\n",
       "  'who',\n",
       "  'first',\n",
       "  'finds',\n",
       "  'himself',\n",
       "  'master',\n",
       "  'of',\n",
       "  'his',\n",
       "  'own',\n",
       "  'establishment',\n",
       "  'were',\n",
       "  'sufficient',\n",
       "  'to',\n",
       "  'absorb',\n",
       "  'all',\n",
       "  'my',\n",
       "  'attention',\n",
       "  'while',\n",
       "  'holmes',\n",
       "  'who',\n",
       "  'loathed',\n",
       "  'every',\n",
       "  'form',\n",
       "  'of',\n",
       "  'society',\n",
       "  'with',\n",
       "  'his',\n",
       "  'whole',\n",
       "  'bohemian',\n",
       "  'soul',\n",
       "  'remained',\n",
       "  'in',\n",
       "  'our',\n",
       "  'lodgings',\n",
       "  'in',\n",
       "  'baker',\n",
       "  'street',\n",
       "  'buried',\n",
       "  'among',\n",
       "  'his',\n",
       "  'old',\n",
       "  'books',\n",
       "  'and',\n",
       "  'alternating',\n",
       "  'from',\n",
       "  'week',\n",
       "  'to',\n",
       "  'week',\n",
       "  'between',\n",
       "  'cocaine',\n",
       "  'and',\n",
       "  'ambition',\n",
       "  'the',\n",
       "  'drowsiness',\n",
       "  'of',\n",
       "  'the',\n",
       "  'drug',\n",
       "  'and',\n",
       "  'the',\n",
       "  'fierce',\n",
       "  'energy',\n",
       "  'of',\n",
       "  'his',\n",
       "  'own',\n",
       "  'keen',\n",
       "  'nature',\n",
       "  'he',\n",
       "  'was',\n",
       "  'still',\n",
       "  'as',\n",
       "  'ever',\n",
       "  'deeply',\n",
       "  'attracted',\n",
       "  'by',\n",
       "  'the',\n",
       "  'study',\n",
       "  'of',\n",
       "  'crime',\n",
       "  'and',\n",
       "  'occupied',\n",
       "  'his',\n",
       "  'immense',\n",
       "  'faculties',\n",
       "  'and',\n",
       "  'extraordinary',\n",
       "  'powers',\n",
       "  'of',\n",
       "  'observation',\n",
       "  'in',\n",
       "  'following',\n",
       "  'out',\n",
       "  'those',\n",
       "  'clews',\n",
       "  'and',\n",
       "  'clearing',\n",
       "  'up',\n",
       "  'those',\n",
       "  'mysteries',\n",
       "  'which',\n",
       "  'had',\n",
       "  'been',\n",
       "  'abandoned',\n",
       "  'as',\n",
       "  'hopeless',\n",
       "  'by',\n",
       "  'the',\n",
       "  'official',\n",
       "  'police',\n",
       "  'from',\n",
       "  'time',\n",
       "  'to',\n",
       "  'time',\n",
       "  'i',\n",
       "  'heard',\n",
       "  'some',\n",
       "  'vague',\n",
       "  'account',\n",
       "  'of',\n",
       "  'his',\n",
       "  'doings',\n",
       "  'of',\n",
       "  'his',\n",
       "  'summons',\n",
       "  'to',\n",
       "  'odessa',\n",
       "  'in',\n",
       "  'the',\n",
       "  'case',\n",
       "  'of',\n",
       "  'the',\n",
       "  'trepoff',\n",
       "  'murder',\n",
       "  'of',\n",
       "  'his',\n",
       "  'clearing',\n",
       "  'up',\n",
       "  'of',\n",
       "  'the',\n",
       "  'singular',\n",
       "  'tragedy',\n",
       "  'of',\n",
       "  'the',\n",
       "  'atkinson',\n",
       "  'brothers',\n",
       "  'at',\n",
       "  'trincomalee',\n",
       "  'and',\n",
       "  'finally',\n",
       "  'of',\n",
       "  'the',\n",
       "  'mission',\n",
       "  'which',\n",
       "  'he',\n",
       "  'had',\n",
       "  'accomplished',\n",
       "  'so',\n",
       "  'delicately',\n",
       "  'and',\n",
       "  'successfully',\n",
       "  'for',\n",
       "  'the',\n",
       "  'reigning',\n",
       "  'family',\n",
       "  'of',\n",
       "  'holland',\n",
       "  'beyond',\n",
       "  'these',\n",
       "  'signs',\n",
       "  'of',\n",
       "  'his',\n",
       "  'activity',\n",
       "  'however',\n",
       "  'which',\n",
       "  'i',\n",
       "  'merely',\n",
       "  'shared',\n",
       "  'with',\n",
       "  'all',\n",
       "  'the',\n",
       "  'readers',\n",
       "  'of',\n",
       "  'the',\n",
       "  'daily',\n",
       "  'press',\n",
       "  'i',\n",
       "  'knew',\n",
       "  'little',\n",
       "  'of',\n",
       "  'my',\n",
       "  'former',\n",
       "  'friend',\n",
       "  'and',\n",
       "  'companion',\n",
       "  'one',\n",
       "  'night--it',\n",
       "  'was',\n",
       "  'on',\n",
       "  'the',\n",
       "  'th',\n",
       "  'of',\n",
       "  'march',\n",
       "  'i',\n",
       "  'was',\n",
       "  'returning',\n",
       "  'from',\n",
       "  'a',\n",
       "  'journey',\n",
       "  'to',\n",
       "  'a',\n",
       "  'patient',\n",
       "  'for',\n",
       "  'i',\n",
       "  'had',\n",
       "  'now',\n",
       "  'returned',\n",
       "  'to',\n",
       "  'civil',\n",
       "  'practice',\n",
       "  'when',\n",
       "  'my',\n",
       "  'way',\n",
       "  'led',\n",
       "  'me',\n",
       "  'through',\n",
       "  'baker',\n",
       "  'street',\n",
       "  'as',\n",
       "  'i',\n",
       "  'passed',\n",
       "  'the',\n",
       "  'well-remembered',\n",
       "  'door',\n",
       "  'which',\n",
       "  'must',\n",
       "  'always',\n",
       "  'be',\n",
       "  'associated',\n",
       "  'in',\n",
       "  'my',\n",
       "  'mind',\n",
       "  'with',\n",
       "  'my',\n",
       "  'wooing',\n",
       "  'and',\n",
       "  'with',\n",
       "  'the',\n",
       "  'dark',\n",
       "  'incidents',\n",
       "  'of',\n",
       "  'the',\n",
       "  'study',\n",
       "  'in',\n",
       "  'scarlet',\n",
       "  'i',\n",
       "  'was',\n",
       "  'seized',\n",
       "  'with',\n",
       "  'a',\n",
       "  'keen',\n",
       "  'desire',\n",
       "  'to',\n",
       "  'see',\n",
       "  'holmes',\n",
       "  'again',\n",
       "  'and',\n",
       "  'to',\n",
       "  'know',\n",
       "  'how',\n",
       "  'he',\n",
       "  'was',\n",
       "  'employing',\n",
       "  'his',\n",
       "  'extraordinary',\n",
       "  'powers',\n",
       "  'his',\n",
       "  'rooms',\n",
       "  'were',\n",
       "  'brilliantly',\n",
       "  'lighted',\n",
       "  'and',\n",
       "  'even',\n",
       "  'as',\n",
       "  'i',\n",
       "  'looked',\n",
       "  'up',\n",
       "  'i',\n",
       "  'saw',\n",
       "  'his',\n",
       "  'tall',\n",
       "  'spare',\n",
       "  'figure',\n",
       "  'pass',\n",
       "  'twice',\n",
       "  'in',\n",
       "  'a',\n",
       "  'dark',\n",
       "  'silhouette',\n",
       "  'against',\n",
       "  'the',\n",
       "  'blind',\n",
       "  'he',\n",
       "  'was',\n",
       "  'pacing',\n",
       "  'the',\n",
       "  'room',\n",
       "  'swiftly',\n",
       "  'eagerly',\n",
       "  'with',\n",
       "  'his',\n",
       "  'head',\n",
       "  'sunk',\n",
       "  'upon',\n",
       "  'his',\n",
       "  'chest',\n",
       "  'and',\n",
       "  'his',\n",
       "  'hands',\n",
       "  'clasped',\n",
       "  'behind',\n",
       "  'him',\n",
       "  'to',\n",
       "  'me',\n",
       "  'who',\n",
       "  'knew',\n",
       "  'his',\n",
       "  'every',\n",
       "  'mood',\n",
       "  'and',\n",
       "  'habit',\n",
       "  'his',\n",
       "  'attitude',\n",
       "  'and',\n",
       "  'manner',\n",
       "  'told',\n",
       "  'their',\n",
       "  'own',\n",
       "  'story',\n",
       "  'he',\n",
       "  'was',\n",
       "  'at',\n",
       "  'work',\n",
       "  'again',\n",
       "  'he',\n",
       "  'had',\n",
       "  'risen',\n",
       "  'out',\n",
       "  'of',\n",
       "  'his',\n",
       "  'drug-created',\n",
       "  'dreams',\n",
       "  'and',\n",
       "  'was',\n",
       "  'hot',\n",
       "  'upon',\n",
       "  'the',\n",
       "  'scent',\n",
       "  'of',\n",
       "  'some',\n",
       "  'new',\n",
       "  'problem',\n",
       "  'i',\n",
       "  'rang',\n",
       "  'the',\n",
       "  'bell',\n",
       "  'and',\n",
       "  'was',\n",
       "  'shown',\n",
       "  'up',\n",
       "  'to',\n",
       "  'the',\n",
       "  'chamber',\n",
       "  'which',\n",
       "  'had',\n",
       "  'formerly',\n",
       "  'been',\n",
       "  'in',\n",
       "  'part',\n",
       "  'my',\n",
       "  'own',\n",
       "  'his',\n",
       "  'manner',\n",
       "  'was',\n",
       "  'not',\n",
       "  'effusive',\n",
       "  'it',\n",
       "  'seldom',\n",
       "  'was',\n",
       "  'but',\n",
       "  'he',\n",
       "  'was',\n",
       "  'glad',\n",
       "  'i',\n",
       "  'think',\n",
       "  'to',\n",
       "  'see',\n",
       "  'me',\n",
       "  'with',\n",
       "  'hardly',\n",
       "  'a',\n",
       "  'word',\n",
       "  'spoken',\n",
       "  'but',\n",
       "  'with',\n",
       "  'a',\n",
       "  'kindly',\n",
       "  'eye',\n",
       "  'he',\n",
       "  'waved',\n",
       "  'me',\n",
       "  'to',\n",
       "  'an',\n",
       "  'armchair',\n",
       "  'threw',\n",
       "  'across',\n",
       "  'his',\n",
       "  'case',\n",
       "  'of',\n",
       "  'cigars',\n",
       "  'and',\n",
       "  'indicated',\n",
       "  'a',\n",
       "  'spirit',\n",
       "  'case',\n",
       "  'and',\n",
       "  'a',\n",
       "  'gasogene',\n",
       "  'in',\n",
       "  'the',\n",
       "  'corner',\n",
       "  'then',\n",
       "  'he',\n",
       "  'stood',\n",
       "  'before',\n",
       "  'the',\n",
       "  'fire',\n",
       "  'and',\n",
       "  'looked',\n",
       "  'me',\n",
       "  'over',\n",
       "  'in',\n",
       "  'his',\n",
       "  'singular',\n",
       "  'introspective',\n",
       "  'fashion',\n",
       "  'wedlock',\n",
       "  'suits',\n",
       "  'you',\n",
       "  'he',\n",
       "  'remarked',\n",
       "  'i',\n",
       "  'think',\n",
       "  'watson',\n",
       "  'that',\n",
       "  'you',\n",
       "  'have',\n",
       "  'put',\n",
       "  'on',\n",
       "  'seven',\n",
       "  'and',\n",
       "  'a',\n",
       "  'half',\n",
       "  'pounds',\n",
       "  'since',\n",
       "  'i',\n",
       "  'saw',\n",
       "  'you',\n",
       "  'seven',\n",
       "  'i',\n",
       "  'answered',\n",
       "  'indeed',\n",
       "  'i',\n",
       "  'should',\n",
       "  'have',\n",
       "  'thought',\n",
       "  'a',\n",
       "  'little',\n",
       "  'more',\n",
       "  'just',\n",
       "  'a',\n",
       "  'trifle',\n",
       "  'more',\n",
       "  'i',\n",
       "  'fancy',\n",
       "  'watson',\n",
       "  'and',\n",
       "  'in',\n",
       "  'practice',\n",
       "  'again',\n",
       "  'i',\n",
       "  'observe',\n",
       "  'you',\n",
       "  'did',\n",
       "  'not',\n",
       "  'tell',\n",
       "  'me',\n",
       "  'that',\n",
       "  'you',\n",
       "  'intended',\n",
       "  'to',\n",
       "  'go',\n",
       "  'into',\n",
       "  'harness',\n",
       "  'then',\n",
       "  'how',\n",
       "  'do',\n",
       "  'you',\n",
       "  'know',\n",
       "  'i',\n",
       "  'see',\n",
       "  'it',\n",
       "  'i',\n",
       "  'deduce',\n",
       "  'it',\n",
       "  'how',\n",
       "  'do',\n",
       "  'i',\n",
       "  'know',\n",
       "  'that',\n",
       "  'you',\n",
       "  'have',\n",
       "  'been',\n",
       "  'getting',\n",
       "  'yourself',\n",
       "  'very',\n",
       "  'wet',\n",
       "  'lately',\n",
       "  'and',\n",
       "  'that',\n",
       "  'you',\n",
       "  'have',\n",
       "  'a',\n",
       "  'most',\n",
       "  'clumsy',\n",
       "  'and',\n",
       "  'careless',\n",
       "  'servant',\n",
       "  'girl',\n",
       "  'my',\n",
       "  'dear',\n",
       "  'holmes',\n",
       "  'said',\n",
       "  'i',\n",
       "  'this',\n",
       "  'is',\n",
       "  'too',\n",
       "  'much',\n",
       "  'you',\n",
       "  'would',\n",
       "  'certainly',\n",
       "  'have',\n",
       "  'been',\n",
       "  'burned',\n",
       "  'had',\n",
       "  'you',\n",
       "  'lived',\n",
       "  'a',\n",
       "  'few',\n",
       "  'centuries',\n",
       "  'ago',\n",
       "  'it',\n",
       "  'is',\n",
       "  'true',\n",
       "  'that',\n",
       "  'i',\n",
       "  'had',\n",
       "  'a',\n",
       "  'country',\n",
       "  'walk',\n",
       "  'on',\n",
       "  'thursday',\n",
       "  'and',\n",
       "  'came',\n",
       "  'home',\n",
       "  'in',\n",
       "  'a',\n",
       "  'dreadful',\n",
       "  'mess',\n",
       "  'but',\n",
       "  'as',\n",
       "  'i',\n",
       "  'have',\n",
       "  'changed',\n",
       "  'my',\n",
       "  'clothes',\n",
       "  'i',\n",
       "  \"can't\",\n",
       "  'imagine',\n",
       "  'how',\n",
       "  'you',\n",
       "  'deduce',\n",
       "  'it',\n",
       "  'as',\n",
       "  'to',\n",
       "  'mary',\n",
       "  'jane',\n",
       "  'she',\n",
       "  'is',\n",
       "  'incorrigible',\n",
       "  'and',\n",
       "  'my',\n",
       "  'wife',\n",
       "  'has',\n",
       "  'given',\n",
       "  'her',\n",
       "  'notice',\n",
       "  'but',\n",
       "  'there',\n",
       "  'again',\n",
       "  'i',\n",
       "  'fail',\n",
       "  'to',\n",
       "  'see',\n",
       "  'how',\n",
       "  'you',\n",
       "  'work',\n",
       "  'it',\n",
       "  'out',\n",
       "  'he',\n",
       "  'chuckled',\n",
       "  'to',\n",
       "  'himself',\n",
       "  'and',\n",
       "  'rubbed',\n",
       "  'his',\n",
       "  'long',\n",
       "  'nervous',\n",
       "  'hands',\n",
       "  'together',\n",
       "  'it',\n",
       "  'is',\n",
       "  'simplicity',\n",
       "  'itself',\n",
       "  'said',\n",
       "  'he',\n",
       "  'my',\n",
       "  'eyes',\n",
       "  'tell',\n",
       "  'me',\n",
       "  'that',\n",
       "  'on',\n",
       "  'the',\n",
       "  'inside',\n",
       "  'of',\n",
       "  'your',\n",
       "  'left',\n",
       "  'shoe',\n",
       "  'just',\n",
       "  'where',\n",
       "  'the',\n",
       "  'firelight',\n",
       "  'strikes',\n",
       "  'it',\n",
       "  'the',\n",
       "  'leather',\n",
       "  'is',\n",
       "  'scored',\n",
       "  'by',\n",
       "  'six',\n",
       "  'almost',\n",
       "  'parallel',\n",
       "  'cuts',\n",
       "  'obviously',\n",
       "  'they',\n",
       "  'have',\n",
       "  'been',\n",
       "  'caused',\n",
       "  'by',\n",
       "  'someone',\n",
       "  'who',\n",
       "  'has',\n",
       "  'very',\n",
       "  'carelessly',\n",
       "  'scraped',\n",
       "  'round',\n",
       "  'the',\n",
       "  'edges',\n",
       "  'of',\n",
       "  'the',\n",
       "  'sole',\n",
       "  'in',\n",
       "  'order',\n",
       "  'to',\n",
       "  'remove',\n",
       "  'crusted',\n",
       "  'mud',\n",
       "  'from',\n",
       "  'it',\n",
       "  'hence',\n",
       "  'you',\n",
       "  'see',\n",
       "  'my',\n",
       "  'double',\n",
       "  'deduction',\n",
       "  'that',\n",
       "  'you',\n",
       "  'had',\n",
       "  'been',\n",
       "  'out',\n",
       "  'in',\n",
       "  'vile',\n",
       "  'weather',\n",
       "  'and',\n",
       "  'that',\n",
       "  'you',\n",
       "  'had',\n",
       "  'a',\n",
       "  'particularly',\n",
       "  'malignant',\n",
       "  'boot-slicking',\n",
       "  'specimen',\n",
       "  'of',\n",
       "  'the',\n",
       "  'london',\n",
       "  'slavey',\n",
       "  'as',\n",
       "  'to',\n",
       "  'your',\n",
       "  'practice',\n",
       "  'if',\n",
       "  'a',\n",
       "  'gentleman',\n",
       "  'walks',\n",
       "  'into',\n",
       "  'my',\n",
       "  'rooms',\n",
       "  'smelling',\n",
       "  'of',\n",
       "  'iodoform',\n",
       "  'with',\n",
       "  'a',\n",
       "  'black',\n",
       "  'mark',\n",
       "  'of',\n",
       "  'nitrate',\n",
       "  'of',\n",
       "  'silver',\n",
       "  'upon',\n",
       "  'his',\n",
       "  'right',\n",
       "  'forefinger',\n",
       "  'and',\n",
       "  'a',\n",
       "  'bulge',\n",
       "  'on',\n",
       "  'the',\n",
       "  'side',\n",
       "  'of',\n",
       "  'his',\n",
       "  'top',\n",
       "  'hat',\n",
       "  'to',\n",
       "  'show',\n",
       "  'where',\n",
       "  'he',\n",
       "  'has',\n",
       "  'secreted',\n",
       "  'his',\n",
       "  ...]]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doc_tokens = [list(pre.tokenize(txt)) for txt in list(corpus_txt)]\n",
    "doc_tokens[:1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Create Dictionaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "id_types, doc_ids = pre.create_dictionaries(doc_labels, doc_tokens)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Sparse BOW Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sparse_bow = pre.create_mm(doc_labels, doc_tokens, id_types, doc_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>doc_id</th>\n",
       "      <th>token_id</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"30\" valign=\"top\">0</th>\n",
       "      <th>8192</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8193</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12972</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23895</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12301</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16398</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11608</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20499</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12310</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>819</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8219</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7514</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20510</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4129</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19803</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20517</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20518</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16424</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8236</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4141</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4142</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4144</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20529</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8244</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"30\" valign=\"top\">16</th>\n",
       "      <th>11595</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12228</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20422</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24521</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8696</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20429</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20430</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12137</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>271</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20472</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4050</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18002</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20802</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8153</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19639</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24543</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9137</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12258</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24547</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20454</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12264</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20458</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15698</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4081</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4094</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12279</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12280</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12281</th>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12285</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20478</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>58745 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 0\n",
       "doc_id token_id   \n",
       "0      8192      1\n",
       "       8193      2\n",
       "       2         1\n",
       "       12972     1\n",
       "       23895     2\n",
       "       12        4\n",
       "       12301     1\n",
       "       16398     2\n",
       "       16        1\n",
       "       11608     1\n",
       "       20499     2\n",
       "       21        2\n",
       "       12310     3\n",
       "       23        1\n",
       "       819       1\n",
       "       8219      1\n",
       "       7514      3\n",
       "       20510     1\n",
       "       4129      3\n",
       "       19803     2\n",
       "       20517     2\n",
       "       20518     1\n",
       "       16424     1\n",
       "       8236      2\n",
       "       4141      1\n",
       "       4142      2\n",
       "       4144      1\n",
       "       20529     1\n",
       "       8244      1\n",
       "       53        1\n",
       "...             ..\n",
       "16     11595     1\n",
       "       12228     1\n",
       "       20422     1\n",
       "       24521     1\n",
       "       8696      1\n",
       "       20429     1\n",
       "       20430     1\n",
       "       12137     1\n",
       "       271       1\n",
       "       20472     2\n",
       "       4050      1\n",
       "       18002     4\n",
       "       20802     1\n",
       "       8153      1\n",
       "       19639     2\n",
       "       24543     1\n",
       "       9137      1\n",
       "       12258     2\n",
       "       24547     1\n",
       "       20454     1\n",
       "       12264     2\n",
       "       20458     1\n",
       "       15698     1\n",
       "       4081      2\n",
       "       4094      1\n",
       "       12279     2\n",
       "       12280     1\n",
       "       12281     6\n",
       "       12285     2\n",
       "       20478     1\n",
       "\n",
       "[58745 rows x 1 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sparse_bow"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Speichern der Market Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'num_docs = max(sparse_bow.index.get_level_values(\"doc_id\"))\\nnum_types = max(sparse_bow.index.get_level_values(\"token_id\"))\\nsum_counts = sum(sparse_bow[0])\\n\\nheader_string = str(num_docs) + \" \" + str(num_types) + \" \" + str(sum_counts) + \"\\n\"\\n\\nwith open(\"gb_plain.mm\", \\'a\\', encoding = \"utf-8\") as f:\\n    f.write(\"%%MatrixMarket matrix coordinate real general\\n\")\\n    #f.write(header_string)\\n    sparse_bow.to_csv( f, sep = \\' \\', header = None)'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''num_docs = max(sparse_bow.index.get_level_values(\"doc_id\"))\n",
    "num_types = max(sparse_bow.index.get_level_values(\"token_id\"))\n",
    "sum_counts = sum(sparse_bow[0])\n",
    "\n",
    "header_string = str(num_docs) + \" \" + str(num_types) + \" \" + str(sum_counts) + \"\\n\"\n",
    "\n",
    "with open(\"gb_plain.mm\", 'a', encoding = \"utf-8\") as f:\n",
    "    f.write(\"%%MatrixMarket matrix coordinate real general\\n\")\n",
    "    #f.write(header_string)\n",
    "    sparse_bow.to_csv( f, sep = ' ', header = None)'''"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Convert to Doc-Topic Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "MemoryError",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mMemoryError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-24-5e34d998a83e>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      7\u001b[0m \u001b[1;31m#idx = [type2id[key] for key in set(sparse_bow.index.get_level_values(\"token_id\"))]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      8\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 9\u001b[1;33m \u001b[0mset\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msparse_bow\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_level_values\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"token_id\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     10\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     11\u001b[0m \u001b[1;31m#doctopic_matrix = pd.DataFrame(columns=cols, index=idx)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mMemoryError\u001b[0m: "
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "doc2id = {value : key for key, value in doc_ids.items()}\n",
    "type2id = {value : key for key, value in id_types.items()}\n",
    "\n",
    "cols = [doc2id[key] for key in set(sparse_bow.index.get_level_values(\"doc_id\"))]\n",
    "#idx = [type2id[key] for key in set(sparse_bow.index.get_level_values(\"token_id\"))]\n",
    "\n",
    "set(sparse_bow.index.get_level_values(\"token_id\"))\n",
    "\n",
    "#doctopic_matrix = pd.DataFrame(columns=cols, index=idx)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Remove Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os.path\n",
    "basepath =  os.path.abspath(os.path.join(\".\", os.pardir))\n",
    "\n",
    "with open(os.path.join(basepath, \"Topics/tutorial_supplementals/stopwords/en\"), 'r', encoding = 'utf-8') as f: \n",
    "    stopword_list = f.read().split('\\n')\n",
    "    \n",
    "stopword_list = set(stopword_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sparse_df_stopwords_removed = pre.remove_features(sparse_bow, id_types, stopword_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print(len(sparse_bow))\n",
    "print(len(sparse_df_stopwords_removed))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Create Feature Remove Lists"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "stopword_list = pre.find_stopwords(docterm_matrix, 100)\n",
    "stopword_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "hapax = set(docterm_matrix[docterm_matrix == 1].index)\n",
    "#hapax = set(docterm_matrix[(docterm_matrix < 2).all(axis=1)].index)\n",
    "len(hapax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "hapax_list = collection.find_hapax(docterm_matrix)\n",
    "len(hapax_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "feature_list = stopword_list.union(hapax_list)\n",
    "print(feature_list)\n",
    "clean_term_frequency = docterm_matrix.drop(feature_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Remove Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "feature_list = stopword_list.union(hapax_list)\n",
    "clean_term_frequency = collection.remove_features(docterm_matrix, feature_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Lemmata anhand POS-Tags auswählen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lemmas = collection.filter_POS_tags(corpus_csv)\n",
    "next(lemmas)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Topic Modeling with Gensim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "corpusPath = \"gb_plain.mm\"\n",
    "\n",
    "mm = MmCorpus(corpusPath) #import momentan in visual \n",
    "# -> da ich mir noch nicht sicher bin, welche Funktionen in das tm_gensim.py sollen\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#in case you're only loading the corpus - build dict first\n",
    "\n",
    "import pickle\n",
    "\n",
    "pickle.dump( id_types, open( \"gb_plain.dictionary\", \"wb\" ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "id_types = pickle.load(open(\"gb_plain.dictionary\", 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#mm = gensim.corpora.MmCorpus(\"gb_all.mm\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#id_types = pickle.load(open(\"gb_all.dictionary\", 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "model = LdaModel(corpus=mm, id2word=id_types, num_topics=60, alpha = \"symmetric\", passes = 10) #import momentan in visual \n",
    "# -> da ich mir noch nicht sicher bin, welche Funktionen in das tm_gensim.py sollen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "topic_nr_x = model.get_topic_terms(10)\n",
    "\n",
    "[id_types[i[0]] for i in topic_nr_x]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "topics = model.show_topics(num_topics = 60)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "topics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Topic Modeling mit Mallet"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Mallet Binary erzeugen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import os.path\n",
    "basepath =  os.path.abspath(os.path.join(\".\", os.pardir))\n",
    "basepath"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#path_to_mallet = \"/home/sina/Uni/Dariah/mallet/bin/mallet\"\n",
    "path_to_mallet = \"mallet\"\n",
    "path_to_corpus = os.path.join(basepath, \"Topics\\\\corpus_txt\")\n",
    "\n",
    "malletBinary, outfolder = mal.create_mallet_binary(path_to_corpus, path_to_mallet)\n",
    "\n",
    "#toDo absolute Pfade"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Mallet output erzeugen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "path_to_mallet = \"/home/sina/Uni/Dariah/mallet/bin/mallet\"\n",
    "malletBinary = \"/home/sina/Uni/Dariah/DARIAH-Topics/Topics/mallet_output/malletBinary.mallet\"\n",
    "mal.create_mallet_model(malletBinary, outfolder, path_to_mallet)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Visualisierung "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "lda_model = 'out_easy/corpus.lda'\n",
    "corpus = 'out_easy/corpus.mm'\n",
    "dictionary = 'out_easy/corpus.dict'\n",
    "doc_labels = 'out_easy/corpus_doclabels.txt'\n",
    "interactive  = False\n",
    "\n",
    "vis = visual.Visualization(lda_model, corpus, dictionary, doc_labels, interactive)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Heatmap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "heatmap = visual.make_heatmap()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "visual.save_heatmap(\"./visualizations/heatmap\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Interactive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vis = collection.Visualization(lda_model, corpus, dictionary, doc_labels, interactive=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vis.make_interactive()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vis.save_interactive(\"./visualizations/interactive\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Ob ihr wirklich richtig steht, seht ihr, wenn ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "![success](http://cdn2.hubspot.net/hub/128506/file-446943132-jpg/images/computer_woman_success.jpg)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}