IntegrationTest_v01.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/funcy/decorators.py:56: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead\n",
      "  spec = inspect.getargspec(func)\n",
      "/usr/local/lib/python3.5/dist-packages/funcy/decorators.py:56: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead\n",
      "  spec = inspect.getargspec(func)\n",
      "/usr/local/lib/python3.5/dist-packages/funcy/decorators.py:56: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead\n",
      "  spec = inspect.getargspec(func)\n",
      "/usr/local/lib/python3.5/dist-packages/funcy/decorators.py:56: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead\n",
      "  spec = inspect.getargspec(func)\n",
      "/usr/local/lib/python3.5/dist-packages/funcy/decorators.py:56: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead\n",
      "  spec = inspect.getargspec(func)\n",
      "/usr/local/lib/python3.5/dist-packages/funcy/decorators.py:56: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead\n",
      "  spec = inspect.getargspec(func)\n"
     ]
    }
   ],
   "source": [
    "from dariah_topics import preprocessing as pre\n",
    "from dariah_topics import visualization as visual\n",
    "from dariah_topics import mallet as mal\n",
    "from dariah_topics import evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Liste mit Dateinamen erzeugen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['corpus_txt/Poe_EurekaAProsePoem.txt',\n",
       " 'corpus_txt/Howard_TheDevilinIron.txt',\n",
       " 'corpus_txt/Lovecraft_TheShunnedHouse.txt',\n",
       " 'corpus_txt/Howard_SchadowsinZamboula.txt',\n",
       " 'corpus_txt/Doyle_AStudyinScarlet.txt']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path_txt = \"corpus_txt\"\n",
    "#path_txt = \"grenzbote_plain/*/\"\n",
    "\n",
    "doclist_txt = pre.create_document_list(path_txt)\n",
    "assert doclist_txt, \"No documents found\"\n",
    "doclist_txt[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['corpus_csv/Howard_GodsoftheNorth.txt.csv',\n",
       " 'corpus_csv/Poe_EurekaAProsePoem.txt.csv',\n",
       " 'corpus_csv/Poe_TheMasqueoftheRedDeath.txt.csv',\n",
       " 'corpus_csv/Poe_ThePurloinedLetter.txt.csv',\n",
       " 'corpus_csv/Howard_ShadowsintheMoonlight.txt.csv']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path_csv = \"corpus_csv\"\n",
    "\n",
    "doclist_csv = pre.create_document_list(path_csv, 'csv')\n",
    "doclist_csv[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#####  Liste mit Dokumentenlabels erzeugen - (Funktion wird durch Thorsten's generischere Funktion ersetzt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['corpus_txt/Poe_EurekaAProsePoem.txt',\n",
       " 'corpus_txt/Howard_TheDevilinIron.txt',\n",
       " 'corpus_txt/Lovecraft_TheShunnedHouse.txt',\n",
       " 'corpus_txt/Howard_SchadowsinZamboula.txt',\n",
       " 'corpus_txt/Doyle_AStudyinScarlet.txt']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doc_labels = list(pre.get_labels(doclist_txt))\n",
    "doc_labels[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Corpus laden"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "corpus_txt = pre.read_from_txt(doclist_txt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "corpus_csv = pre.read_from_csv(doclist_csv)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Tokenisieren"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['eureka', 'prose', 'poem', 'by', 'edgar']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doc_tokens = [list(pre.tokenize(txt)) for txt in list(corpus_txt)]\n",
    "doc_tokens[0][:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Create Dictionaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "id_types, doc_ids = pre.create_dictionaries(doc_labels, doc_tokens)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Sparse BOW Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sparse_bow = pre.create_mm(doc_labels, doc_tokens, id_types, doc_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>doc_id</th>\n",
       "      <th>token_id</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">1</th>\n",
       "      <th>8192</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8194</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6827</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8202</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 0\n",
       "doc_id token_id   \n",
       "1      8192      1\n",
       "       8194      1\n",
       "       6827      3\n",
       "       5         1\n",
       "       8202      3"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sparse_bow[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Convert to Doc-Topic Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'import pandas as pd\\n\\ndoc2id = {value : key for key, value in doc_ids.items()}\\ntype2id = {value : key for key, value in id_types.items()}\\n\\ncols = [doc2id[key] for key in set(sparse_bow.index.get_level_values(\"doc_id\"))]\\n#idx = [type2id[key] for key in set(sparse_bow.index.get_level_values(\"token_id\"))]\\n\\nset(sparse_bow.index.get_level_values(\"token_id\"))\\n\\n#doctopic_matrix = pd.DataFrame(columns=cols, index=idx)'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''import pandas as pd\n",
    "\n",
    "doc2id = {value : key for key, value in doc_ids.items()}\n",
    "type2id = {value : key for key, value in id_types.items()}\n",
    "\n",
    "cols = [doc2id[key] for key in set(sparse_bow.index.get_level_values(\"doc_id\"))]\n",
    "#idx = [type2id[key] for key in set(sparse_bow.index.get_level_values(\"token_id\"))]\n",
    "\n",
    "set(sparse_bow.index.get_level_values(\"token_id\"))\n",
    "\n",
    "#doctopic_matrix = pd.DataFrame(columns=cols, index=idx)'''"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Remove Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os.path\n",
    "basepath = os.path.abspath('.')\n",
    "\n",
    "with open(os.path.join(basepath, \"tutorial_supplementals\", \"stopwords\", \"en.txt\"), 'r', encoding = 'utf-8') as f: \n",
    "    stopword_list = f.read().split('\\n')\n",
    "    \n",
    "stopword_list = set(stopword_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sparse_df_stopwords_removed = pre.remove_features(sparse_bow, id_types, stopword_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "50942\n",
      "45754\n"
     ]
    }
   ],
   "source": [
    "print(len(sparse_bow))\n",
    "print(len(sparse_df_stopwords_removed))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Create Feature Remove Lists"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "100"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stopword_list = pre.find_stopwords(sparse_bow, id_types, 100)\n",
    "len(stopword_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "17746"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hapax_list = pre.find_hapax(sparse_bow, id_types)\n",
    "len(hapax_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Remove Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "feature_list = set(stopword_list).union(hapax_list)\n",
    "clean_term_frequency = pre.remove_features(sparse_bow, id_types, feature_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4892"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(clean_term_frequency)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Topic Modeling with Gensim"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Saving Sparse BOW"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "num_docs = max(sparse_bow.index.get_level_values(\"doc_id\"))\n",
    "num_types = max(sparse_bow.index.get_level_values(\"token_id\"))\n",
    "sum_counts = sum(sparse_bow[0])\n",
    "\n",
    "header_string = str(num_docs) + \" \" + str(num_types) + \" \" + str(sum_counts) + \"\\n\"\n",
    "\n",
    "with open(\"gb_plain.mm\", 'w', encoding = \"utf-8\") as f:\n",
    "    pass\n",
    "\n",
    "with open(\"gb_plain.mm\", 'a', encoding = \"utf-8\") as f:\n",
    "    f.write(\"%%MatrixMarket matrix coordinate real general\\n\")\n",
    "    f.write(header_string)\n",
    "    sparse_bow.to_csv( f, sep = ' ', header = None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Import Market Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from gensim.models import LdaModel\n",
    "from gensim.corpora import MmCorpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "mm = MmCorpus(\"gb_plain.mm\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#in case you're only loading the corpus - build dict first\n",
    "\n",
    "#import pickle\n",
    "\n",
    "#pickle.dump( id_types, open( \"gb_plain.dictionary\", \"wb\" ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#id_types = pickle.load(open(\"gb_plain.dictionary\", 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#mm = gensim.corpora.MmCorpus(\"gb_all.mm\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#id_types = pickle.load(open(\"gb_all.dictionary\", 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "doc2id = {value : key for key, value in doc_ids.items()}\n",
    "type2id = {value : key for key, value in id_types.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING gensim.models.ldamodel: too few updates, training might not converge; consider increasing the number of passes or iterations to improve accuracy\n"
     ]
    }
   ],
   "source": [
    "#model = LdaModel(corpus=mm, id2word=type2id, num_topics=60, alpha = \"symmetric\", passes = 10) #import momentan in visual \n",
    "# -> da ich mir noch nicht sicher bin, welche Funktionen in das tm_gensim.py sollen\n",
    "model = LdaModel(corpus=mm, id2word=type2id, num_topics=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['grasses',\n",
       " 'quench',\n",
       " 'replacement',\n",
       " \"creature's\",\n",
       " 'butterflies',\n",
       " 'flagship',\n",
       " 'struggles',\n",
       " 'cow-kick',\n",
       " 'subterrene',\n",
       " 'swam']"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Anzeige der keywords für topic n\n",
    "n = 1\n",
    "topic_nr_x = model.get_topic_terms(n)\n",
    "\n",
    "topicTerms = [type2id[i[0]] for i in topic_nr_x]\n",
    "topicTerms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "19964\n"
     ]
    }
   ],
   "source": [
    "#Anzeige der id des ersten keywords aus dem topic\n",
    "'''n = 0\n",
    "for key, value in type2id.items():\n",
    "    if value == topicTerms[n]:\n",
    "        print(key)'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(19964, 0.045547697399961583),\n",
       " (7484, 0.02555935157378144),\n",
       " (20844, 0.021748152184078273),\n",
       " (12390, 0.018450207185275385),\n",
       " (17775, 0.014793747674321884),\n",
       " (2813, 0.010719961603461841),\n",
       " (17527, 0.010692766393465185),\n",
       " (10804, 0.0090416682617215326),\n",
       " (7691, 0.0087376172189662857),\n",
       " (901, 0.008659425952339133)]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Gensim-Funktion -> Anzeige des word_id-probability-mappings eines topics n\n",
    "'''n = 0\n",
    "topicN = model.get_topic_terms(n)\n",
    "topicN'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0,\n",
       "  '0.046*\"grasses\" + 0.026*\"replacement\" + 0.022*\"quench\" + 0.018*\"creature\\'s\" + 0.015*\"struggles\" + 0.011*\"flagship\" + 0.011*\"butterflies\" + 0.009*\"swam\" + 0.009*\"cow-kick\" + 0.009*\"packs\"'),\n",
       " (1,\n",
       "  '0.074*\"grasses\" + 0.028*\"quench\" + 0.026*\"replacement\" + 0.023*\"creature\\'s\" + 0.016*\"butterflies\" + 0.016*\"flagship\" + 0.015*\"struggles\" + 0.011*\"cow-kick\" + 0.011*\"subterrene\" + 0.010*\"swam\"')]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Topic probabilities\n",
    "topics = model.show_topics(num_topics = 10)\n",
    "topics[:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>48.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50912</th>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50913</th>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50914</th>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50915</th>\n",
       "      <td>13.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50916</th>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50917</th>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50918</th>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50919</th>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50920</th>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50921</th>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50922</th>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50923</th>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50924</th>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50925</th>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50926</th>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50927</th>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50928</th>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50929</th>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50930</th>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50931</th>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50932</th>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50933</th>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50934</th>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50935</th>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50936</th>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50937</th>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50938</th>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50939</th>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50940</th>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50941</th>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>50942 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          0\n",
       "0       1.0\n",
       "1       1.0\n",
       "2       3.0\n",
       "3       1.0\n",
       "4       3.0\n",
       "5       1.0\n",
       "6       3.0\n",
       "7       2.0\n",
       "8       1.0\n",
       "9       4.0\n",
       "10      5.0\n",
       "11      1.0\n",
       "12      1.0\n",
       "13      1.0\n",
       "14      1.0\n",
       "15      1.0\n",
       "16      2.0\n",
       "17      3.0\n",
       "18      1.0\n",
       "19      3.0\n",
       "20      3.0\n",
       "21      2.0\n",
       "22      1.0\n",
       "23      1.0\n",
       "24      2.0\n",
       "25      2.0\n",
       "26      1.0\n",
       "27      1.0\n",
       "28     48.0\n",
       "29      6.0\n",
       "...     ...\n",
       "50912   4.0\n",
       "50913   1.0\n",
       "50914   1.0\n",
       "50915  13.0\n",
       "50916   1.0\n",
       "50917   1.0\n",
       "50918   1.0\n",
       "50919   1.0\n",
       "50920   2.0\n",
       "50921   1.0\n",
       "50922   1.0\n",
       "50923   1.0\n",
       "50924   1.0\n",
       "50925   1.0\n",
       "50926   1.0\n",
       "50927   2.0\n",
       "50928   2.0\n",
       "50929   3.0\n",
       "50930   3.0\n",
       "50931   1.0\n",
       "50932   2.0\n",
       "50933   1.0\n",
       "50934   2.0\n",
       "50935   1.0\n",
       "50936   1.0\n",
       "50937   1.0\n",
       "50938   1.0\n",
       "50939   1.0\n",
       "50940   4.0\n",
       "50941   1.0\n",
       "\n",
       "[50942 rows x 1 columns]"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from scipy.io import mmread\n",
    "test = mmread(\"gb_plain.mm\")\n",
    "import pandas as pd\n",
    "pd.DataFrame(test.data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Topic Modeling mit Mallet"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Mallet Binary erzeugen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'tutorial_supplementals/mallet_output/malletBinary.mallet'"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''import os\n",
    "\n",
    "#Variablen zum Testen\n",
    "path_to_mallet = \"/home/sina/Uni/Dariah/mallet/bin/mallet\"\n",
    "\n",
    "path_to_corpus = os.path.join(os.path.abspath('.'), 'corpus_txt')\n",
    "assert os.path.exists(path_to_corpus)\n",
    "\n",
    "malletBinary = mal.create_mallet_binary(path_to_corpus, path_to_mallet)\n",
    "malletBinary'''"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Mallet output erzeugen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "'''outfolder = os.path.join(os.path.abspath('.'), \"tutorial_supplementals/mallet_output\")\n",
    "mal.create_mallet_model(malletBinary,outfolder)'''"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Matrix von Mallet output erzeugen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "'''basepath = os.path.join(os.path.abspath('.'), \"tutorial_supplementals/mallet_output\")\n",
    "doc_topics = os.path.join(basepath, \"doc_topics.txt\")'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "'''doctopicsMatrix = mal.create_MalletMatrix(doc_topics)'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "      <th>18</th>\n",
       "      <th>19</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.018470</td>\n",
       "      <td>0.012259</td>\n",
       "      <td>0.069794</td>\n",
       "      <td>0.008336</td>\n",
       "      <td>0.023374</td>\n",
       "      <td>0.380353</td>\n",
       "      <td>0.007028</td>\n",
       "      <td>0.088754</td>\n",
       "      <td>0.003759</td>\n",
       "      <td>0.058352</td>\n",
       "      <td>0.008336</td>\n",
       "      <td>0.037757</td>\n",
       "      <td>0.009971</td>\n",
       "      <td>0.000817</td>\n",
       "      <td>0.004740</td>\n",
       "      <td>0.017816</td>\n",
       "      <td>0.145636</td>\n",
       "      <td>0.092677</td>\n",
       "      <td>0.003432</td>\n",
       "      <td>0.008336</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.022766</td>\n",
       "      <td>0.020562</td>\n",
       "      <td>0.091914</td>\n",
       "      <td>0.011052</td>\n",
       "      <td>0.015209</td>\n",
       "      <td>0.158291</td>\n",
       "      <td>0.004125</td>\n",
       "      <td>0.098274</td>\n",
       "      <td>0.019932</td>\n",
       "      <td>0.078248</td>\n",
       "      <td>0.006140</td>\n",
       "      <td>0.026922</td>\n",
       "      <td>0.006266</td>\n",
       "      <td>0.001984</td>\n",
       "      <td>0.001669</td>\n",
       "      <td>0.209302</td>\n",
       "      <td>0.142925</td>\n",
       "      <td>0.075225</td>\n",
       "      <td>0.004755</td>\n",
       "      <td>0.004440</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.007416</td>\n",
       "      <td>0.019088</td>\n",
       "      <td>0.106867</td>\n",
       "      <td>0.002845</td>\n",
       "      <td>0.279410</td>\n",
       "      <td>0.115718</td>\n",
       "      <td>0.003331</td>\n",
       "      <td>0.086636</td>\n",
       "      <td>0.016705</td>\n",
       "      <td>0.053227</td>\n",
       "      <td>0.007513</td>\n",
       "      <td>0.023902</td>\n",
       "      <td>0.005763</td>\n",
       "      <td>0.000559</td>\n",
       "      <td>0.001094</td>\n",
       "      <td>0.013398</td>\n",
       "      <td>0.170622</td>\n",
       "      <td>0.079974</td>\n",
       "      <td>0.003039</td>\n",
       "      <td>0.002894</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.015372</td>\n",
       "      <td>0.026422</td>\n",
       "      <td>0.080110</td>\n",
       "      <td>0.217192</td>\n",
       "      <td>0.031297</td>\n",
       "      <td>0.138739</td>\n",
       "      <td>0.003087</td>\n",
       "      <td>0.082710</td>\n",
       "      <td>0.014202</td>\n",
       "      <td>0.079981</td>\n",
       "      <td>0.006207</td>\n",
       "      <td>0.022197</td>\n",
       "      <td>0.005882</td>\n",
       "      <td>0.001722</td>\n",
       "      <td>0.001852</td>\n",
       "      <td>0.016737</td>\n",
       "      <td>0.179948</td>\n",
       "      <td>0.068151</td>\n",
       "      <td>0.004062</td>\n",
       "      <td>0.004127</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.002207</td>\n",
       "      <td>0.536885</td>\n",
       "      <td>0.131463</td>\n",
       "      <td>0.004098</td>\n",
       "      <td>0.005990</td>\n",
       "      <td>0.002207</td>\n",
       "      <td>0.072194</td>\n",
       "      <td>0.012926</td>\n",
       "      <td>0.030580</td>\n",
       "      <td>0.053909</td>\n",
       "      <td>0.005990</td>\n",
       "      <td>0.065259</td>\n",
       "      <td>0.002207</td>\n",
       "      <td>0.005359</td>\n",
       "      <td>0.006620</td>\n",
       "      <td>0.007251</td>\n",
       "      <td>0.041299</td>\n",
       "      <td>0.006620</td>\n",
       "      <td>0.001576</td>\n",
       "      <td>0.005359</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.009872</td>\n",
       "      <td>0.107346</td>\n",
       "      <td>0.168356</td>\n",
       "      <td>0.005603</td>\n",
       "      <td>0.001512</td>\n",
       "      <td>0.011473</td>\n",
       "      <td>0.377535</td>\n",
       "      <td>0.016987</td>\n",
       "      <td>0.018410</td>\n",
       "      <td>0.079420</td>\n",
       "      <td>0.005425</td>\n",
       "      <td>0.112327</td>\n",
       "      <td>0.007560</td>\n",
       "      <td>0.002935</td>\n",
       "      <td>0.004358</td>\n",
       "      <td>0.005069</td>\n",
       "      <td>0.053273</td>\n",
       "      <td>0.011117</td>\n",
       "      <td>0.000445</td>\n",
       "      <td>0.000978</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.005495</td>\n",
       "      <td>0.237705</td>\n",
       "      <td>0.169789</td>\n",
       "      <td>0.003513</td>\n",
       "      <td>0.003873</td>\n",
       "      <td>0.008377</td>\n",
       "      <td>0.249955</td>\n",
       "      <td>0.018465</td>\n",
       "      <td>0.021348</td>\n",
       "      <td>0.070528</td>\n",
       "      <td>0.007836</td>\n",
       "      <td>0.108719</td>\n",
       "      <td>0.002252</td>\n",
       "      <td>0.009638</td>\n",
       "      <td>0.008557</td>\n",
       "      <td>0.003333</td>\n",
       "      <td>0.056837</td>\n",
       "      <td>0.010178</td>\n",
       "      <td>0.002072</td>\n",
       "      <td>0.001531</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.003606</td>\n",
       "      <td>0.135263</td>\n",
       "      <td>0.164109</td>\n",
       "      <td>0.002311</td>\n",
       "      <td>0.000462</td>\n",
       "      <td>0.015440</td>\n",
       "      <td>0.328310</td>\n",
       "      <td>0.029863</td>\n",
       "      <td>0.047245</td>\n",
       "      <td>0.055936</td>\n",
       "      <td>0.012666</td>\n",
       "      <td>0.100314</td>\n",
       "      <td>0.001942</td>\n",
       "      <td>0.006749</td>\n",
       "      <td>0.019323</td>\n",
       "      <td>0.001387</td>\n",
       "      <td>0.057045</td>\n",
       "      <td>0.014885</td>\n",
       "      <td>0.001757</td>\n",
       "      <td>0.001387</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.404977</td>\n",
       "      <td>0.023547</td>\n",
       "      <td>0.086005</td>\n",
       "      <td>0.007515</td>\n",
       "      <td>0.007849</td>\n",
       "      <td>0.042585</td>\n",
       "      <td>0.008183</td>\n",
       "      <td>0.049933</td>\n",
       "      <td>0.013527</td>\n",
       "      <td>0.122077</td>\n",
       "      <td>0.006847</td>\n",
       "      <td>0.022879</td>\n",
       "      <td>0.007849</td>\n",
       "      <td>0.016199</td>\n",
       "      <td>0.006847</td>\n",
       "      <td>0.003841</td>\n",
       "      <td>0.136774</td>\n",
       "      <td>0.026887</td>\n",
       "      <td>0.003173</td>\n",
       "      <td>0.002505</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.011169</td>\n",
       "      <td>0.058814</td>\n",
       "      <td>0.063583</td>\n",
       "      <td>0.002117</td>\n",
       "      <td>0.000365</td>\n",
       "      <td>0.010877</td>\n",
       "      <td>0.003236</td>\n",
       "      <td>0.009904</td>\n",
       "      <td>0.003042</td>\n",
       "      <td>0.232115</td>\n",
       "      <td>0.002166</td>\n",
       "      <td>0.005816</td>\n",
       "      <td>0.000511</td>\n",
       "      <td>0.419773</td>\n",
       "      <td>0.001192</td>\n",
       "      <td>0.001679</td>\n",
       "      <td>0.164858</td>\n",
       "      <td>0.005086</td>\n",
       "      <td>0.002798</td>\n",
       "      <td>0.000900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.011158</td>\n",
       "      <td>0.020926</td>\n",
       "      <td>0.038397</td>\n",
       "      <td>0.003057</td>\n",
       "      <td>0.000993</td>\n",
       "      <td>0.027200</td>\n",
       "      <td>0.001946</td>\n",
       "      <td>0.025770</td>\n",
       "      <td>0.006631</td>\n",
       "      <td>0.162524</td>\n",
       "      <td>0.002660</td>\n",
       "      <td>0.008140</td>\n",
       "      <td>0.007346</td>\n",
       "      <td>0.006234</td>\n",
       "      <td>0.000437</td>\n",
       "      <td>0.001946</td>\n",
       "      <td>0.139414</td>\n",
       "      <td>0.016320</td>\n",
       "      <td>0.515208</td>\n",
       "      <td>0.003693</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0.001598</td>\n",
       "      <td>0.009564</td>\n",
       "      <td>0.038716</td>\n",
       "      <td>0.000948</td>\n",
       "      <td>0.001598</td>\n",
       "      <td>0.004525</td>\n",
       "      <td>0.001761</td>\n",
       "      <td>0.114684</td>\n",
       "      <td>0.217583</td>\n",
       "      <td>0.017421</td>\n",
       "      <td>0.061095</td>\n",
       "      <td>0.062016</td>\n",
       "      <td>0.005337</td>\n",
       "      <td>0.001111</td>\n",
       "      <td>0.338797</td>\n",
       "      <td>0.000732</td>\n",
       "      <td>0.068139</td>\n",
       "      <td>0.046193</td>\n",
       "      <td>0.001002</td>\n",
       "      <td>0.007180</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>0.012863</td>\n",
       "      <td>0.024182</td>\n",
       "      <td>0.047232</td>\n",
       "      <td>0.008541</td>\n",
       "      <td>0.008541</td>\n",
       "      <td>0.044556</td>\n",
       "      <td>0.001749</td>\n",
       "      <td>0.071311</td>\n",
       "      <td>0.182239</td>\n",
       "      <td>0.027063</td>\n",
       "      <td>0.054847</td>\n",
       "      <td>0.103416</td>\n",
       "      <td>0.268471</td>\n",
       "      <td>0.001955</td>\n",
       "      <td>0.007306</td>\n",
       "      <td>0.009981</td>\n",
       "      <td>0.070694</td>\n",
       "      <td>0.044762</td>\n",
       "      <td>0.003396</td>\n",
       "      <td>0.006894</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>0.001204</td>\n",
       "      <td>0.001686</td>\n",
       "      <td>0.008913</td>\n",
       "      <td>0.001067</td>\n",
       "      <td>0.000585</td>\n",
       "      <td>0.001549</td>\n",
       "      <td>0.000654</td>\n",
       "      <td>0.039539</td>\n",
       "      <td>0.023641</td>\n",
       "      <td>0.003200</td>\n",
       "      <td>0.269270</td>\n",
       "      <td>0.006228</td>\n",
       "      <td>0.002168</td>\n",
       "      <td>0.000516</td>\n",
       "      <td>0.000516</td>\n",
       "      <td>0.001204</td>\n",
       "      <td>0.022953</td>\n",
       "      <td>0.203889</td>\n",
       "      <td>0.000241</td>\n",
       "      <td>0.410977</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0.010281</td>\n",
       "      <td>0.026515</td>\n",
       "      <td>0.065476</td>\n",
       "      <td>0.013528</td>\n",
       "      <td>0.011364</td>\n",
       "      <td>0.039502</td>\n",
       "      <td>0.007035</td>\n",
       "      <td>0.053571</td>\n",
       "      <td>0.028680</td>\n",
       "      <td>0.062229</td>\n",
       "      <td>0.007035</td>\n",
       "      <td>0.076299</td>\n",
       "      <td>0.292749</td>\n",
       "      <td>0.008117</td>\n",
       "      <td>0.017857</td>\n",
       "      <td>0.023268</td>\n",
       "      <td>0.126082</td>\n",
       "      <td>0.110931</td>\n",
       "      <td>0.005952</td>\n",
       "      <td>0.013528</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>0.010365</td>\n",
       "      <td>0.071570</td>\n",
       "      <td>0.089339</td>\n",
       "      <td>0.002468</td>\n",
       "      <td>0.005429</td>\n",
       "      <td>0.063672</td>\n",
       "      <td>0.012340</td>\n",
       "      <td>0.022211</td>\n",
       "      <td>0.029121</td>\n",
       "      <td>0.019250</td>\n",
       "      <td>0.011352</td>\n",
       "      <td>0.434847</td>\n",
       "      <td>0.018263</td>\n",
       "      <td>0.005429</td>\n",
       "      <td>0.004442</td>\n",
       "      <td>0.007404</td>\n",
       "      <td>0.082428</td>\n",
       "      <td>0.066634</td>\n",
       "      <td>0.009378</td>\n",
       "      <td>0.034057</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>0.007480</td>\n",
       "      <td>0.003644</td>\n",
       "      <td>0.029344</td>\n",
       "      <td>0.005562</td>\n",
       "      <td>0.003260</td>\n",
       "      <td>0.094937</td>\n",
       "      <td>0.005178</td>\n",
       "      <td>0.121404</td>\n",
       "      <td>0.012083</td>\n",
       "      <td>0.028193</td>\n",
       "      <td>0.017069</td>\n",
       "      <td>0.017453</td>\n",
       "      <td>0.292482</td>\n",
       "      <td>0.002877</td>\n",
       "      <td>0.004795</td>\n",
       "      <td>0.005562</td>\n",
       "      <td>0.075758</td>\n",
       "      <td>0.191983</td>\n",
       "      <td>0.004028</td>\n",
       "      <td>0.076908</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          0         1         2         3         4         5         6   \\\n",
       "0   0.018470  0.012259  0.069794  0.008336  0.023374  0.380353  0.007028   \n",
       "1   0.022766  0.020562  0.091914  0.011052  0.015209  0.158291  0.004125   \n",
       "2   0.007416  0.019088  0.106867  0.002845  0.279410  0.115718  0.003331   \n",
       "3   0.015372  0.026422  0.080110  0.217192  0.031297  0.138739  0.003087   \n",
       "4   0.002207  0.536885  0.131463  0.004098  0.005990  0.002207  0.072194   \n",
       "5   0.009872  0.107346  0.168356  0.005603  0.001512  0.011473  0.377535   \n",
       "6   0.005495  0.237705  0.169789  0.003513  0.003873  0.008377  0.249955   \n",
       "7   0.003606  0.135263  0.164109  0.002311  0.000462  0.015440  0.328310   \n",
       "8   0.404977  0.023547  0.086005  0.007515  0.007849  0.042585  0.008183   \n",
       "9   0.011169  0.058814  0.063583  0.002117  0.000365  0.010877  0.003236   \n",
       "10  0.011158  0.020926  0.038397  0.003057  0.000993  0.027200  0.001946   \n",
       "11  0.001598  0.009564  0.038716  0.000948  0.001598  0.004525  0.001761   \n",
       "12  0.012863  0.024182  0.047232  0.008541  0.008541  0.044556  0.001749   \n",
       "13  0.001204  0.001686  0.008913  0.001067  0.000585  0.001549  0.000654   \n",
       "14  0.010281  0.026515  0.065476  0.013528  0.011364  0.039502  0.007035   \n",
       "15  0.010365  0.071570  0.089339  0.002468  0.005429  0.063672  0.012340   \n",
       "16  0.007480  0.003644  0.029344  0.005562  0.003260  0.094937  0.005178   \n",
       "\n",
       "          7         8         9         10        11        12        13  \\\n",
       "0   0.088754  0.003759  0.058352  0.008336  0.037757  0.009971  0.000817   \n",
       "1   0.098274  0.019932  0.078248  0.006140  0.026922  0.006266  0.001984   \n",
       "2   0.086636  0.016705  0.053227  0.007513  0.023902  0.005763  0.000559   \n",
       "3   0.082710  0.014202  0.079981  0.006207  0.022197  0.005882  0.001722   \n",
       "4   0.012926  0.030580  0.053909  0.005990  0.065259  0.002207  0.005359   \n",
       "5   0.016987  0.018410  0.079420  0.005425  0.112327  0.007560  0.002935   \n",
       "6   0.018465  0.021348  0.070528  0.007836  0.108719  0.002252  0.009638   \n",
       "7   0.029863  0.047245  0.055936  0.012666  0.100314  0.001942  0.006749   \n",
       "8   0.049933  0.013527  0.122077  0.006847  0.022879  0.007849  0.016199   \n",
       "9   0.009904  0.003042  0.232115  0.002166  0.005816  0.000511  0.419773   \n",
       "10  0.025770  0.006631  0.162524  0.002660  0.008140  0.007346  0.006234   \n",
       "11  0.114684  0.217583  0.017421  0.061095  0.062016  0.005337  0.001111   \n",
       "12  0.071311  0.182239  0.027063  0.054847  0.103416  0.268471  0.001955   \n",
       "13  0.039539  0.023641  0.003200  0.269270  0.006228  0.002168  0.000516   \n",
       "14  0.053571  0.028680  0.062229  0.007035  0.076299  0.292749  0.008117   \n",
       "15  0.022211  0.029121  0.019250  0.011352  0.434847  0.018263  0.005429   \n",
       "16  0.121404  0.012083  0.028193  0.017069  0.017453  0.292482  0.002877   \n",
       "\n",
       "          14        15        16        17        18        19  \n",
       "0   0.004740  0.017816  0.145636  0.092677  0.003432  0.008336  \n",
       "1   0.001669  0.209302  0.142925  0.075225  0.004755  0.004440  \n",
       "2   0.001094  0.013398  0.170622  0.079974  0.003039  0.002894  \n",
       "3   0.001852  0.016737  0.179948  0.068151  0.004062  0.004127  \n",
       "4   0.006620  0.007251  0.041299  0.006620  0.001576  0.005359  \n",
       "5   0.004358  0.005069  0.053273  0.011117  0.000445  0.000978  \n",
       "6   0.008557  0.003333  0.056837  0.010178  0.002072  0.001531  \n",
       "7   0.019323  0.001387  0.057045  0.014885  0.001757  0.001387  \n",
       "8   0.006847  0.003841  0.136774  0.026887  0.003173  0.002505  \n",
       "9   0.001192  0.001679  0.164858  0.005086  0.002798  0.000900  \n",
       "10  0.000437  0.001946  0.139414  0.016320  0.515208  0.003693  \n",
       "11  0.338797  0.000732  0.068139  0.046193  0.001002  0.007180  \n",
       "12  0.007306  0.009981  0.070694  0.044762  0.003396  0.006894  \n",
       "13  0.000516  0.001204  0.022953  0.203889  0.000241  0.410977  \n",
       "14  0.017857  0.023268  0.126082  0.110931  0.005952  0.013528  \n",
       "15  0.004442  0.007404  0.082428  0.066634  0.009378  0.034057  \n",
       "16  0.004795  0.005562  0.075758  0.191983  0.004028  0.076908  "
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''import pandas as pd\n",
    "pd.DataFrame(doctopicsMatrix)'''"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Visualisierung "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "'''lda_model = 'out_easy/corpus.lda'\n",
    "corpus = 'out_easy/corpus.mm'\n",
    "dictionary = 'out_easy/corpus.dict'\n",
    "doc_labels = 'out_easy/corpus_doclabels.txt'\n",
    "interactive  = False\n",
    "\n",
    "vis = visual.Visualization(lda_model, corpus, dictionary, doc_labels, interactive)'''"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Heatmap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#heatmap = visual.make_heatmap()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#visual.save_heatmap(\"./visualizations/heatmap\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Interactive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#vis = collection.Visualization(lda_model, corpus, dictionary, doc_labels, interactive=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#vis.make_interactive()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#vis.save_interactive(\"./visualizations/interactive\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "segmented_topics = evaluation.topic_segmenter(model, type2id, 60)\n",
    "segmented_topics[0][:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "umass_probability = evaluation.token_probability(doc_tokens, segmented_topics)\n",
    "umass_probability"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "umass = evaluation.calculate_umass(segmented_topics, umass_probability, doc_tokens, 60)\n",
    "umass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "wiki_list = evaluation.wikipedia_table_crawler()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "wiki_corpus = evaluation.wikipedia_crawler(wiki_list, size=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "uci_probability = evaluation.token_probability(wiki_corpus, segmented_topics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pmi = evaluation.calculate_pointwise_mutual_information(segmented_topics, wiki_corpus, uci_probability)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "uci = evaluation.calculate_uci(pmi, wiki_corpus, 60)\n",
    "uci"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "![success](http://cdn2.hubspot.net/hub/128506/file-446943132-jpg/images/computer_woman_success.jpg)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}