IntegrationTest_txt_gensim.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/site-packages/gensim/utils.py:1015: UserWarning: Pattern library is not installed, lemmatization won't be available.\n",
      "  warnings.warn(\"Pattern library is not installed, lemmatization won't be available.\")\n"
     ]
    }
   ],
   "source": [
    "from dariah_topics import preprocessing as pre\n",
    "from dariah_topics import visualization as visual\n",
    "from dariah_topics import evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Liste mit Dateinamen erzeugen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/site-packages/IPython/core/formatters.py:92: DeprecationWarning: DisplayFormatter._ipython_display_formatter_default is deprecated: use @default decorator instead.\n",
      "  def _ipython_display_formatter_default(self):\n",
      "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/site-packages/IPython/core/formatters.py:98: DeprecationWarning: DisplayFormatter._formatters_default is deprecated: use @default decorator instead.\n",
      "  def _formatters_default(self):\n",
      "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/site-packages/IPython/core/formatters.py:677: DeprecationWarning: PlainTextFormatter._deferred_printers_default is deprecated: use @default decorator instead.\n",
      "  def _deferred_printers_default(self):\n",
      "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/site-packages/IPython/core/formatters.py:669: DeprecationWarning: PlainTextFormatter._singleton_printers_default is deprecated: use @default decorator instead.\n",
      "  def _singleton_printers_default(self):\n",
      "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/site-packages/IPython/core/formatters.py:672: DeprecationWarning: PlainTextFormatter._type_printers_default is deprecated: use @default decorator instead.\n",
      "  def _type_printers_default(self):\n",
      "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/site-packages/IPython/core/formatters.py:669: DeprecationWarning: PlainTextFormatter._singleton_printers_default is deprecated: use @default decorator instead.\n",
      "  def _singleton_printers_default(self):\n",
      "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/site-packages/IPython/core/formatters.py:672: DeprecationWarning: PlainTextFormatter._type_printers_default is deprecated: use @default decorator instead.\n",
      "  def _type_printers_default(self):\n",
      "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/site-packages/IPython/core/formatters.py:677: DeprecationWarning: PlainTextFormatter._deferred_printers_default is deprecated: use @default decorator instead.\n",
      "  def _deferred_printers_default(self):\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['corpus_txt/Doyle_AScandalinBohemia.txt',\n",
       " 'corpus_txt/Doyle_AStudyinScarlet.txt',\n",
       " 'corpus_txt/Doyle_TheHoundoftheBaskervilles.txt',\n",
       " 'corpus_txt/Doyle_TheSignoftheFour.txt',\n",
       " 'corpus_txt/Howard_GodsoftheNorth.txt']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path_txt = \"corpus_txt\"\n",
    "#path_txt = \"grenzbote_plain/*/\"\n",
    "#path_txt = \"wiki/\"\n",
    "\n",
    "doclist_txt = pre.create_document_list(path_txt)\n",
    "assert doclist_txt, \"No documents found\"\n",
    "doclist_txt[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#####  Liste mit Dokumentenlabels erzeugen - (Funktion wird durch Thorsten's generischere Funktion ersetzt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['corpus_txt/Doyle_AScandalinBohemia.txt',\n",
       " 'corpus_txt/Doyle_AStudyinScarlet.txt',\n",
       " 'corpus_txt/Doyle_TheHoundoftheBaskervilles.txt',\n",
       " 'corpus_txt/Doyle_TheSignoftheFour.txt',\n",
       " 'corpus_txt/Howard_GodsoftheNorth.txt']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doc_labels = list(pre.get_labels(doclist_txt))\n",
    "doc_labels[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Corpus laden"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "corpus_txt = pre.read_from_txt(doclist_txt)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Tokenisieren"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "doc_tokens = [list(pre.tokenize(txt)) for txt in list(corpus_txt)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Create Dictionaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "id_types, doc_ids = pre.create_dictionaries(doc_labels, doc_tokens)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Sparse BOW Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sparse_bow = pre.create_mm(doc_labels, doc_tokens, id_types, doc_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>doc_id</th>\n",
       "      <th>token_id</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">1</th>\n",
       "      <th>9820</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12291</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12292</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 0\n",
       "doc_id token_id   \n",
       "1      9820      1\n",
       "       2         1\n",
       "       12291     1\n",
       "       12292     3\n",
       "       6         1"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sparse_bow[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Topic Modeling with Gensim"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Saving Sparse BOW"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pre.save_bow_mm(sparse_bow, \"gensim_txt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Import Market Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from gensim.models import LdaModel\n",
    "from gensim.corpora import MmCorpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "mm = MmCorpus(\"gensim_txt.mm\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "doc2id = {value : key for key, value in doc_ids.items()}\n",
    "type2id = {value : key for key, value in id_types.items()}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Remove Features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "type2id = {value : key for key, value in id_types.items()}\n",
    "sparse_bow_collapsed = sparse_bow.groupby(sparse_bow.index.get_level_values('token_id')).sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "sparse_bow_hapax = sparse_bow_collapsed.loc[sparse_bow_collapsed[0] == 1]\n",
    "hapax = [type2id[key] for key in sparse_bow_hapax.index.get_level_values('token_id')]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "len(hapax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os.path\n",
    "basepath = os.path.abspath('.')\n",
    "\n",
    "with open(os.path.join(basepath, \"tutorial_supplementals\", \"stopwords\", \"de.txt\"), 'r', encoding = 'utf-8') as f: \n",
    "    stopword_list = f.read().split('\\n')\n",
    "    \n",
    "stopword_list = set(stopword_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "hapax_from_remove = pre.find_hapax(sparse_bow, id_types)\n",
    "stopwords_from_remove = pre.find_stopwords(sparse_bow, id_types, mfw=75)\n",
    "\n",
    "#features_to_be_removed = set(hapax_from_remove + stopwords_from_remove)\n",
    "features_to_be_removed = stopwords_from_remove\n",
    "\n",
    "sparse_bow_short = pre.remove_features(sparse_bow, id_types, features_to_be_removed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "75"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(features_to_be_removed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pre.save_bow_mm(sparse_bow_short, \"gensim_txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mm = MmCorpus(\"gensim_txt.mm\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Convert sparse_bow to list of (doc, tokens) tuples (like doc2bow)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "doc2bow_list = []\n",
    "\n",
    "for doc in sparse_bow_short.index.groupby(sparse_bow_short.index.get_level_values('doc_id')):\n",
    "    temp = [(token, count) for token, count in zip(sparse_bow_short.loc[doc].index, sparse_bow_short.loc[doc][0])]\n",
    "    doc2bow_list.append(temp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "16"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(doc2bow_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#model = LdaModel(corpus=mm, id2word=type2id, num_topics=60, alpha = \"symmetric\", passes = 10) #import momentan in visual \n",
    "# -> da ich mir noch nicht sicher bin, welche Funktionen in das tm_gensim.py sollen\n",
    "model = LdaModel(corpus=mm, id2word=type2id, num_topics=20, passes = 10, iterations = 20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(1, 0.19637437178315331),\n",
       " (2, 0.090343075140683918),\n",
       " (3, 0.094308762035912083),\n",
       " (4, 0.04870776113704553),\n",
       " (10, 0.14402773517950188),\n",
       " (11, 0.063089538229874467),\n",
       " (13, 0.077610164942319246),\n",
       " (15, 0.089822382311483345),\n",
       " (16, 0.022185781952032801),\n",
       " (17, 0.045392065366853834),\n",
       " (18, 0.11205460065760847)]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.get_document_topics(doc2bow_list[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['glowing',\n",
       " \"beast's\",\n",
       " 'contending',\n",
       " 'real-true',\n",
       " 'dollie',\n",
       " 'cryin',\n",
       " 'confounded',\n",
       " 'nursey',\n",
       " 'charities',\n",
       " 'tuesday']"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Anzeige der keywords für topic n\n",
    "n = 1\n",
    "topic_nr_x = model.get_topic_terms(n)\n",
    "\n",
    "topicTerms = [type2id[i[0]] for i in topic_nr_x]\n",
    "topicTerms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0,\n",
       "  '0.000*\"salient\" + 0.000*\"cryin\" + 0.000*\"dollie\" + 0.000*\"rat-holes\" + 0.000*\"heater\" + 0.000*\"twice\" + 0.000*\"cubbing-times\" + 0.000*\"grabbing\" + 0.000*\"contending\" + 0.000*\"charities\"'),\n",
       " (1,\n",
       "  '0.005*\"glowing\" + 0.004*\"beast\\'s\" + 0.004*\"contending\" + 0.004*\"real-true\" + 0.004*\"dollie\" + 0.003*\"cryin\" + 0.003*\"confounded\" + 0.003*\"nursey\" + 0.003*\"charities\" + 0.003*\"tuesday\"'),\n",
       " (2,\n",
       "  '0.007*\"serpents\" + 0.005*\"richest\" + 0.005*\"commented\" + 0.005*\"treasure-box\" + 0.004*\"hymnbook\" + 0.004*\"goings-on\" + 0.004*\"father\\'s\" + 0.004*\"heater\" + 0.004*\"exhalation\" + 0.004*\"contending\"'),\n",
       " (3,\n",
       "  '0.012*\"system’s\" + 0.005*\"orifice\" + 0.004*\"disintegrated\" + 0.004*\"valuable\" + 0.004*\"salient\" + 0.004*\"performed\" + 0.003*\"treasure-box\" + 0.003*\"brain-fevered\" + 0.003*\"fortune\" + 0.003*\"greetings\"'),\n",
       " (4,\n",
       "  '0.009*\"glowing\" + 0.004*\"pyari\" + 0.004*\"memorably\" + 0.004*\"tuesday\" + 0.004*\"grabbing\" + 0.004*\"propos\" + 0.004*\"ma\" + 0.003*\"racial\" + 0.003*\"kent\\'s\" + 0.003*\"dollie\"'),\n",
       " (5,\n",
       "  '0.000*\"twice\" + 0.000*\"fortune\" + 0.000*\"treasure-box\" + 0.000*\"greetings\" + 0.000*\"salient\" + 0.000*\"charities\" + 0.000*\"cryin\" + 0.000*\"heater\" + 0.000*\"trails\" + 0.000*\"real-true\"'),\n",
       " (6,\n",
       "  '0.007*\"ordinance\" + 0.007*\"tramped\" + 0.006*\"malachi\" + 0.004*\"fortune\" + 0.004*\"smouldered\" + 0.003*\"dependent\" + 0.003*\"civilian\" + 0.003*\"thicker\" + 0.003*\"oasis\" + 0.003*\"partake\"'),\n",
       " (7,\n",
       "  '0.002*\"persecutors\" + 0.001*\"bluntly\" + 0.001*\"pillows\" + 0.001*\"gods\" + 0.001*\"twenty-three\" + 0.001*\"omnipotence\" + 0.001*\"mocked\" + 0.001*\"filigree\" + 0.001*\"newer\" + 0.001*\"harm\"'),\n",
       " (8,\n",
       "  '0.008*\"sitting\" + 0.007*\"subtle\" + 0.005*\"orifice\" + 0.004*\"scourge\" + 0.004*\"disintegrated\" + 0.004*\"field\" + 0.004*\"marge\" + 0.004*\"confounded\" + 0.003*\"somebodies\" + 0.003*\"creak\"'),\n",
       " (9,\n",
       "  '0.000*\"twice\" + 0.000*\"orifice\" + 0.000*\"beast\\'s\" + 0.000*\"cryin\" + 0.000*\"fortune\" + 0.000*\"unequal\" + 0.000*\"salient\" + 0.000*\"civilian\" + 0.000*\"greetings\" + 0.000*\"glowing\"'),\n",
       " (10,\n",
       "  '0.004*\"turning\" + 0.004*\"fortune\" + 0.004*\"greetings\" + 0.004*\"cryin\" + 0.004*\"scowl\" + 0.004*\"unequal\" + 0.003*\"trails\" + 0.003*\"incredulity\" + 0.003*\"orifice\" + 0.003*\"hisself\"'),\n",
       " (11,\n",
       "  '0.007*\"slink\" + 0.005*\"kent\\'s\" + 0.004*\"twice\" + 0.004*\"assumption\" + 0.004*\"surmounting\" + 0.004*\"sensitive\" + 0.003*\"fortune\" + 0.003*\"spacious\" + 0.003*\"penetrant\" + 0.003*\"orifice\"'),\n",
       " (12,\n",
       "  '0.001*\"contending\" + 0.001*\"real-true\" + 0.001*\"beast\\'s\" + 0.000*\"heater\" + 0.000*\"propos\" + 0.000*\"twice\" + 0.000*\"dollie\" + 0.000*\"volunteered\" + 0.000*\"cryin\" + 0.000*\"pyari\"'),\n",
       " (13,\n",
       "  '0.009*\"fortune\" + 0.006*\"civilian\" + 0.005*\"cryin\" + 0.005*\"shutting\" + 0.005*\"peaches\" + 0.005*\"astern\" + 0.004*\"smouldered\" + 0.004*\"memorably\" + 0.004*\"controlled\" + 0.004*\"countenance\"'),\n",
       " (14,\n",
       "  '0.000*\"twice\" + 0.000*\"fortune\" + 0.000*\"treasure-box\" + 0.000*\"charities\" + 0.000*\"real-true\" + 0.000*\"dollie\" + 0.000*\"half-visible\" + 0.000*\"cubbing-times\" + 0.000*\"heater\" + 0.000*\"orifice\"'),\n",
       " (15,\n",
       "  '0.013*\"attains\" + 0.006*\"wall-paper\" + 0.006*\"glowing\" + 0.006*\"uninteresting\" + 0.005*\"volunteered\" + 0.004*\"flee\" + 0.004*\"faintest\" + 0.004*\"twenty-foot\" + 0.004*\"grabbing\" + 0.004*\"salient\"'),\n",
       " (16,\n",
       "  '0.011*\"mast\" + 0.008*\"severing\" + 0.006*\"limitation\" + 0.006*\"kepler\" + 0.006*\"hymnbook\" + 0.005*\"mongoose\\'s\" + 0.004*\"grabbing\" + 0.004*\"truffle\" + 0.004*\"obedient\" + 0.004*\"scorch\"'),\n",
       " (17,\n",
       "  '0.006*\"bewilderment\" + 0.006*\"back-door\" + 0.006*\"repugnance\" + 0.006*\"glowing\" + 0.005*\"feeble\" + 0.005*\"dollie\" + 0.005*\"gaspee\" + 0.005*\"half-visible\" + 0.004*\"hence\" + 0.004*\"twice\"'),\n",
       " (18,\n",
       "  '0.012*\"remarkable\" + 0.008*\"antipodes\" + 0.008*\"weakened\" + 0.007*\"charities\" + 0.006*\"abortive\" + 0.006*\"wave\" + 0.006*\"tarry\" + 0.006*\"salient\" + 0.006*\"ma\" + 0.005*\"unmarried\"'),\n",
       " (19,\n",
       "  '0.000*\"hymnbook\" + 0.000*\"father\\'s\" + 0.000*\"monkey-people\" + 0.000*\"real-true\" + 0.000*\"twice\" + 0.000*\"treasure-box\" + 0.000*\"half-visible\" + 0.000*\"serpents\" + 0.000*\"dollie\" + 0.000*\"heater\"')]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Topic probabilities\n",
    "topics = model.show_topics(num_topics = 40)\n",
    "topics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import regex as re\n",
    "def gensim2dataframe(model):\n",
    "    num_topics = model.num_topics\n",
    "    topics_df = pd.DataFrame(index = range(num_topics), columns= range(10))\n",
    "\n",
    "    topics = model.show_topics(num_topics)\n",
    "    \n",
    "    for topic_dist in topics:    \n",
    "        idx = topic_dist[0]\n",
    "        temp = re.findall(r'\\\"(.+?)\\\"', topics[idx][1])\n",
    "        topics_df.loc[idx] = temp\n",
    "    \n",
    "    return topics_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "topics_df = gensim2dataframe(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "      <th>18</th>\n",
       "      <th>19</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>salient</td>\n",
       "      <td>glowing</td>\n",
       "      <td>serpents</td>\n",
       "      <td>system’s</td>\n",
       "      <td>glowing</td>\n",
       "      <td>twice</td>\n",
       "      <td>ordinance</td>\n",
       "      <td>persecutors</td>\n",
       "      <td>sitting</td>\n",
       "      <td>twice</td>\n",
       "      <td>turning</td>\n",
       "      <td>slink</td>\n",
       "      <td>contending</td>\n",
       "      <td>fortune</td>\n",
       "      <td>twice</td>\n",
       "      <td>attains</td>\n",
       "      <td>mast</td>\n",
       "      <td>bewilderment</td>\n",
       "      <td>remarkable</td>\n",
       "      <td>hymnbook</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>cryin</td>\n",
       "      <td>beast's</td>\n",
       "      <td>richest</td>\n",
       "      <td>orifice</td>\n",
       "      <td>pyari</td>\n",
       "      <td>fortune</td>\n",
       "      <td>tramped</td>\n",
       "      <td>bluntly</td>\n",
       "      <td>subtle</td>\n",
       "      <td>orifice</td>\n",
       "      <td>fortune</td>\n",
       "      <td>kent's</td>\n",
       "      <td>real-true</td>\n",
       "      <td>civilian</td>\n",
       "      <td>fortune</td>\n",
       "      <td>wall-paper</td>\n",
       "      <td>severing</td>\n",
       "      <td>back-door</td>\n",
       "      <td>antipodes</td>\n",
       "      <td>father's</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>dollie</td>\n",
       "      <td>contending</td>\n",
       "      <td>commented</td>\n",
       "      <td>disintegrated</td>\n",
       "      <td>memorably</td>\n",
       "      <td>treasure-box</td>\n",
       "      <td>malachi</td>\n",
       "      <td>pillows</td>\n",
       "      <td>orifice</td>\n",
       "      <td>beast's</td>\n",
       "      <td>greetings</td>\n",
       "      <td>twice</td>\n",
       "      <td>beast's</td>\n",
       "      <td>cryin</td>\n",
       "      <td>treasure-box</td>\n",
       "      <td>glowing</td>\n",
       "      <td>limitation</td>\n",
       "      <td>repugnance</td>\n",
       "      <td>weakened</td>\n",
       "      <td>monkey-people</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>rat-holes</td>\n",
       "      <td>real-true</td>\n",
       "      <td>treasure-box</td>\n",
       "      <td>valuable</td>\n",
       "      <td>tuesday</td>\n",
       "      <td>greetings</td>\n",
       "      <td>fortune</td>\n",
       "      <td>gods</td>\n",
       "      <td>scourge</td>\n",
       "      <td>cryin</td>\n",
       "      <td>cryin</td>\n",
       "      <td>assumption</td>\n",
       "      <td>heater</td>\n",
       "      <td>shutting</td>\n",
       "      <td>charities</td>\n",
       "      <td>uninteresting</td>\n",
       "      <td>kepler</td>\n",
       "      <td>glowing</td>\n",
       "      <td>charities</td>\n",
       "      <td>real-true</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>heater</td>\n",
       "      <td>dollie</td>\n",
       "      <td>hymnbook</td>\n",
       "      <td>salient</td>\n",
       "      <td>grabbing</td>\n",
       "      <td>salient</td>\n",
       "      <td>smouldered</td>\n",
       "      <td>twenty-three</td>\n",
       "      <td>disintegrated</td>\n",
       "      <td>fortune</td>\n",
       "      <td>scowl</td>\n",
       "      <td>surmounting</td>\n",
       "      <td>propos</td>\n",
       "      <td>peaches</td>\n",
       "      <td>real-true</td>\n",
       "      <td>volunteered</td>\n",
       "      <td>hymnbook</td>\n",
       "      <td>feeble</td>\n",
       "      <td>abortive</td>\n",
       "      <td>twice</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>twice</td>\n",
       "      <td>cryin</td>\n",
       "      <td>goings-on</td>\n",
       "      <td>performed</td>\n",
       "      <td>propos</td>\n",
       "      <td>charities</td>\n",
       "      <td>dependent</td>\n",
       "      <td>omnipotence</td>\n",
       "      <td>field</td>\n",
       "      <td>unequal</td>\n",
       "      <td>unequal</td>\n",
       "      <td>sensitive</td>\n",
       "      <td>twice</td>\n",
       "      <td>astern</td>\n",
       "      <td>dollie</td>\n",
       "      <td>flee</td>\n",
       "      <td>mongoose's</td>\n",
       "      <td>dollie</td>\n",
       "      <td>wave</td>\n",
       "      <td>treasure-box</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>cubbing-times</td>\n",
       "      <td>confounded</td>\n",
       "      <td>father's</td>\n",
       "      <td>treasure-box</td>\n",
       "      <td>ma</td>\n",
       "      <td>cryin</td>\n",
       "      <td>civilian</td>\n",
       "      <td>mocked</td>\n",
       "      <td>marge</td>\n",
       "      <td>salient</td>\n",
       "      <td>trails</td>\n",
       "      <td>fortune</td>\n",
       "      <td>dollie</td>\n",
       "      <td>smouldered</td>\n",
       "      <td>half-visible</td>\n",
       "      <td>faintest</td>\n",
       "      <td>grabbing</td>\n",
       "      <td>gaspee</td>\n",
       "      <td>tarry</td>\n",
       "      <td>half-visible</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>grabbing</td>\n",
       "      <td>nursey</td>\n",
       "      <td>heater</td>\n",
       "      <td>brain-fevered</td>\n",
       "      <td>racial</td>\n",
       "      <td>heater</td>\n",
       "      <td>thicker</td>\n",
       "      <td>filigree</td>\n",
       "      <td>confounded</td>\n",
       "      <td>civilian</td>\n",
       "      <td>incredulity</td>\n",
       "      <td>spacious</td>\n",
       "      <td>volunteered</td>\n",
       "      <td>memorably</td>\n",
       "      <td>cubbing-times</td>\n",
       "      <td>twenty-foot</td>\n",
       "      <td>truffle</td>\n",
       "      <td>half-visible</td>\n",
       "      <td>salient</td>\n",
       "      <td>serpents</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>contending</td>\n",
       "      <td>charities</td>\n",
       "      <td>exhalation</td>\n",
       "      <td>fortune</td>\n",
       "      <td>kent's</td>\n",
       "      <td>trails</td>\n",
       "      <td>oasis</td>\n",
       "      <td>newer</td>\n",
       "      <td>somebodies</td>\n",
       "      <td>greetings</td>\n",
       "      <td>orifice</td>\n",
       "      <td>penetrant</td>\n",
       "      <td>cryin</td>\n",
       "      <td>controlled</td>\n",
       "      <td>heater</td>\n",
       "      <td>grabbing</td>\n",
       "      <td>obedient</td>\n",
       "      <td>hence</td>\n",
       "      <td>ma</td>\n",
       "      <td>dollie</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>charities</td>\n",
       "      <td>tuesday</td>\n",
       "      <td>contending</td>\n",
       "      <td>greetings</td>\n",
       "      <td>dollie</td>\n",
       "      <td>real-true</td>\n",
       "      <td>partake</td>\n",
       "      <td>harm</td>\n",
       "      <td>creak</td>\n",
       "      <td>glowing</td>\n",
       "      <td>hisself</td>\n",
       "      <td>orifice</td>\n",
       "      <td>pyari</td>\n",
       "      <td>countenance</td>\n",
       "      <td>orifice</td>\n",
       "      <td>salient</td>\n",
       "      <td>scorch</td>\n",
       "      <td>twice</td>\n",
       "      <td>unmarried</td>\n",
       "      <td>heater</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              0           1             2              3          4   \\\n",
       "0        salient     glowing      serpents       system’s    glowing   \n",
       "1          cryin     beast's       richest        orifice      pyari   \n",
       "2         dollie  contending     commented  disintegrated  memorably   \n",
       "3      rat-holes   real-true  treasure-box       valuable    tuesday   \n",
       "4         heater      dollie      hymnbook        salient   grabbing   \n",
       "5          twice       cryin     goings-on      performed     propos   \n",
       "6  cubbing-times  confounded      father's   treasure-box         ma   \n",
       "7       grabbing      nursey        heater  brain-fevered     racial   \n",
       "8     contending   charities    exhalation        fortune     kent's   \n",
       "9      charities     tuesday    contending      greetings     dollie   \n",
       "\n",
       "             5           6             7              8          9   \\\n",
       "0         twice   ordinance   persecutors        sitting      twice   \n",
       "1       fortune     tramped       bluntly         subtle    orifice   \n",
       "2  treasure-box     malachi       pillows        orifice    beast's   \n",
       "3     greetings     fortune          gods        scourge      cryin   \n",
       "4       salient  smouldered  twenty-three  disintegrated    fortune   \n",
       "5     charities   dependent   omnipotence          field    unequal   \n",
       "6         cryin    civilian        mocked          marge    salient   \n",
       "7        heater     thicker      filigree     confounded   civilian   \n",
       "8        trails       oasis         newer     somebodies  greetings   \n",
       "9     real-true     partake          harm          creak    glowing   \n",
       "\n",
       "            10           11           12           13             14  \\\n",
       "0      turning        slink   contending      fortune          twice   \n",
       "1      fortune       kent's    real-true     civilian        fortune   \n",
       "2    greetings        twice      beast's        cryin   treasure-box   \n",
       "3        cryin   assumption       heater     shutting      charities   \n",
       "4        scowl  surmounting       propos      peaches      real-true   \n",
       "5      unequal    sensitive        twice       astern         dollie   \n",
       "6       trails      fortune       dollie   smouldered   half-visible   \n",
       "7  incredulity     spacious  volunteered    memorably  cubbing-times   \n",
       "8      orifice    penetrant        cryin   controlled         heater   \n",
       "9      hisself      orifice        pyari  countenance        orifice   \n",
       "\n",
       "              15          16            17          18             19  \n",
       "0        attains        mast  bewilderment  remarkable       hymnbook  \n",
       "1     wall-paper    severing     back-door   antipodes       father's  \n",
       "2        glowing  limitation    repugnance    weakened  monkey-people  \n",
       "3  uninteresting      kepler       glowing   charities      real-true  \n",
       "4    volunteered    hymnbook        feeble    abortive          twice  \n",
       "5           flee  mongoose's        dollie        wave   treasure-box  \n",
       "6       faintest    grabbing        gaspee       tarry   half-visible  \n",
       "7    twenty-foot     truffle  half-visible     salient       serpents  \n",
       "8       grabbing    obedient         hence          ma         dollie  \n",
       "9        salient      scorch         twice   unmarried         heater  "
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doc_topic = topics_df.T\n",
    "doc_topic"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Visualisierung "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "'''lda_model = 'out_easy/corpus.lda'\n",
    "corpus = 'out_easy/corpus.mm'\n",
    "dictionary = 'out_easy/corpus.dict'\n",
    "doc_labels = 'out_easy/corpus_doclabels.txt'\n",
    "interactive  = False\n",
    "\n",
    "vis = visual.Visualization(lda_model, corpus, dictionary, doc_labels, interactive)'''"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Create a document-topic matrix (that is a pandas data frame actually)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "doc_topic = visual.create_doc_topic(mm, model, doc_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Visualize document-topic matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "visual.doc_topic_heatmap(doc_topic)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Visualize topic distribution in a single document"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "visual.plot_doc_topics(doc_topic, 0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "![success](http://cdn2.hubspot.net/hub/128506/file-446943132-jpg/images/computer_woman_success.jpg)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}