IntegrationTest_Mallet_Export.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from dariah_topics import preprocessing as pre\n",
    "from dariah_topics import visualization as visual\n",
    "from dariah_topics import evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "#### Liste mit Dateinamen erzeugen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO preprocessing: Creating document list from TXT files ...\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['corpus_txt/Lovecraft_AttheMountainofMadness.txt',\n",
       " 'corpus_txt/Howard_TheDevilinIron.txt',\n",
       " 'corpus_txt/Poe_ThePurloinedLetter.txt',\n",
       " 'corpus_txt/Lovecraft_TheShunnedHouse.txt',\n",
       " 'corpus_txt/Poe_TheMasqueoftheRedDeath.txt']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path_txt = \"corpus_txt\"\n",
    "\n",
    "doclist_txt = pre.create_document_list(path_txt)\n",
    "assert doclist_txt, \"No documents found\"\n",
    "doclist_txt[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "#####  Liste mit Dokumentenlabels erzeugen - (Funktion wird durch Thorsten's generischere Funktion ersetzt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO preprocessing: Creating document labels ...\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['Lovecraft_AttheMountainofMadness',\n",
       " 'Howard_TheDevilinIron',\n",
       " 'Poe_ThePurloinedLetter',\n",
       " 'Lovecraft_TheShunnedHouse',\n",
       " 'Poe_TheMasqueoftheRedDeath']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doc_labels = list(pre.get_labels(doclist_txt))\n",
    "doc_labels[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "#### Corpus laden"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "corpus_txt = pre.read_from_txt(doclist_txt)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "#### Tokenisieren"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO preprocessing: Accessing TXT documents ...\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "17"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doc_tokens = [list(pre.tokenize(txt)) for txt in list(corpus_txt)]\n",
    "len(doc_tokens)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "#### Create Dictionaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "id_types = pre.create_dictionary(doc_tokens)\n",
    "doc_ids = pre.create_dictionary(doc_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "#### Sparse BOW Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "sparse_bow = pre.create_sparse_bow(doc_labels, doc_tokens, id_types, doc_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>doc_id</th>\n",
       "      <th>token_id</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">1</th>\n",
       "      <th>8195</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8197</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8198</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16393</th>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  0\n",
       "doc_id token_id    \n",
       "1      8195       1\n",
       "       8197       1\n",
       "       8198       2\n",
       "       8         11\n",
       "       16393     19"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sparse_bow[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "##### Remove Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import os.path\n",
    "basepath = os.path.abspath('.')\n",
    "\n",
    "with open(os.path.join(basepath, \"tutorial_supplementals\", \"stopwords\", \"en.txt\"), 'r', encoding = 'utf-8') as f: \n",
    "    stopword_list = f.read().split('\\n')\n",
    "    \n",
    "stopword_list = set(stopword_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO preprocessing: Finding hapax legomena ...\n",
      "INFO preprocessing: Finding stopwords ...\n",
      "INFO preprocessing: Removing features ...\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "75"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hapax_from_remove = pre.find_hapax(sparse_bow, id_types)\n",
    "stopwords_from_remove = pre.find_stopwords(sparse_bow, id_types, mfw=75)\n",
    "\n",
    "features_to_be_removed = stopwords_from_remove + hapax_from_remove\n",
    "\n",
    "sparse_bow_short = pre.remove_features(sparse_bow, id_types, features_to_be_removed)\n",
    "len(stopwords_from_remove)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "17"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doc2bow_list = []\n",
    "for doc in sparse_bow.index.groupby(sparse_bow.index.get_level_values('doc_id')):\n",
    "    temp = [(token, count) for token, count in zip(sparse_bow.loc[doc].index, sparse_bow.loc[doc][0])]\n",
    "    doc2bow_list.append(temp)\n",
    "len(doc2bow_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9862"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(features_to_be_removed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "39907"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(doc_tokens[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def remove_features_from_file(doc_token_list, features_to_be_removed):\n",
    "    \"\"\"Removes features using feature list.\n",
    "\n",
    "    Description:\n",
    "        With this function you can remove features from ppreprocessed files. \n",
    "        Commit a list of features.\n",
    "        Use the function `tokenize()` to access your files.\n",
    "\n",
    "    Args:\n",
    "        doc_token_list Union(list[str], str): List of all documents in the corpus\n",
    "            and their tokens.\n",
    "        features_to_be_removed list[str]: List of features that should be\n",
    "        removed\n",
    "    Yields:\n",
    "        cleaned token array\n",
    "\n",
    "    Todo:\n",
    "\n",
    "    Example:\n",
    "        >>> doc_tokens = [['short', 'example', 'example', 'text', 'text']]\n",
    "        >>> features_to_be_removed = ['example']\n",
    "        >>> test = remove_features_from_file(doc_tokens, features_to_be_removed)\n",
    "        >>> list(test)\n",
    "        [['short', 'text', 'text']]\n",
    "    \"\"\"\n",
    "    #log.info(\"Removing features ...\")\n",
    "    doc_token_array = np.array(doc_token_list)\n",
    "    feature_array = np.array(features_to_be_removed)\n",
    "    #get indices of features that should be deleted\n",
    "    indices = np.where(np.in1d(doc_token_array, feature_array,))\n",
    "    doc_token_array = np.delete(doc_token_array, indices)\n",
    "    yield doc_token_array.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "doc_tokens_cleaned = []\n",
    "for doc_token_list in doc_tokens:\n",
    "    doc_token_list_cleaned = remove_features_from_file(doc_token_list, features_to_be_removed)\n",
    "    doc_tokens_cleaned.append(list(doc_token_list_cleaned))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['mountains',\n",
       "  'madness',\n",
       "  'lovecraft',\n",
       "  'chapter',\n",
       "  'am',\n",
       "  'forced',\n",
       "  'speech',\n",
       "  'because',\n",
       "  'men',\n",
       "  'science',\n",
       "  'refused',\n",
       "  'follow',\n",
       "  'advice',\n",
       "  'without',\n",
       "  'knowing',\n",
       "  'why',\n",
       "  'altogether',\n",
       "  'against',\n",
       "  'tell',\n",
       "  'reasons',\n",
       "  'contemplated',\n",
       "  'invasion',\n",
       "  'antarctic',\n",
       "  'its',\n",
       "  'vast',\n",
       "  'fossil',\n",
       "  'hunt',\n",
       "  'its',\n",
       "  'wholesale',\n",
       "  'boring',\n",
       "  'melting',\n",
       "  'ancient',\n",
       "  'ice',\n",
       "  'caps',\n",
       "  'am',\n",
       "  'reluctant',\n",
       "  'because',\n",
       "  'warning',\n",
       "  'may',\n",
       "  'vain',\n",
       "  'doubt',\n",
       "  'real',\n",
       "  'facts',\n",
       "  'must',\n",
       "  'reveal',\n",
       "  'inevitable',\n",
       "  'yet',\n",
       "  'suppressed',\n",
       "  'seem',\n",
       "  'extravagant',\n",
       "  'incredible',\n",
       "  'nothing',\n",
       "  'left',\n",
       "  'hitherto',\n",
       "  'withheld',\n",
       "  'photographs',\n",
       "  'both',\n",
       "  'ordinary',\n",
       "  'aerial',\n",
       "  'count',\n",
       "  'favor',\n",
       "  'damnably',\n",
       "  'vivid',\n",
       "  'graphic',\n",
       "  'still',\n",
       "  'doubted',\n",
       "  'because',\n",
       "  'great',\n",
       "  'lengths',\n",
       "  'clever',\n",
       "  'can',\n",
       "  'carried',\n",
       "  'ink',\n",
       "  'drawings',\n",
       "  'course',\n",
       "  'jeered',\n",
       "  'obvious',\n",
       "  'notwithstanding',\n",
       "  'strangeness',\n",
       "  'technique',\n",
       "  'art',\n",
       "  'ought',\n",
       "  'remark',\n",
       "  'puzzle',\n",
       "  'end',\n",
       "  'must',\n",
       "  'rely',\n",
       "  'judgment',\n",
       "  'standing',\n",
       "  'few',\n",
       "  'scientific',\n",
       "  'leaders',\n",
       "  'hand',\n",
       "  'sufficient',\n",
       "  'independence',\n",
       "  'thought',\n",
       "  'weigh',\n",
       "  'data',\n",
       "  'its',\n",
       "  'own',\n",
       "  'hideously',\n",
       "  'merits',\n",
       "  'light',\n",
       "  'certain',\n",
       "  'primordial',\n",
       "  'highly',\n",
       "  'baffling',\n",
       "  'myth',\n",
       "  'cycles',\n",
       "  'other',\n",
       "  'hand',\n",
       "  'sufficient',\n",
       "  'influence',\n",
       "  'deter',\n",
       "  'exploring',\n",
       "  'world',\n",
       "  'general',\n",
       "  'rash',\n",
       "  'program',\n",
       "  'region',\n",
       "  'those',\n",
       "  'mountains',\n",
       "  'madness',\n",
       "  'unfortunate',\n",
       "  'fact',\n",
       "  'relatively',\n",
       "  'obscure',\n",
       "  'men',\n",
       "  'myself',\n",
       "  'associates',\n",
       "  'connected',\n",
       "  'only',\n",
       "  'small',\n",
       "  'university',\n",
       "  'chance',\n",
       "  'making',\n",
       "  'impression',\n",
       "  'where',\n",
       "  'matters',\n",
       "  'wildly',\n",
       "  'bizarre',\n",
       "  'highly',\n",
       "  'controversial',\n",
       "  'nature',\n",
       "  'concerned',\n",
       "  'further',\n",
       "  'against',\n",
       "  'strictest',\n",
       "  'sense',\n",
       "  'specialists',\n",
       "  'fields',\n",
       "  'came',\n",
       "  'primarily',\n",
       "  'concerned',\n",
       "  'geologist',\n",
       "  'object',\n",
       "  'leading',\n",
       "  'miskatonic',\n",
       "  'university',\n",
       "  'expedition',\n",
       "  'wholly',\n",
       "  'securing',\n",
       "  'specimens',\n",
       "  'rock',\n",
       "  'soil',\n",
       "  'various',\n",
       "  'parts',\n",
       "  'antarctic',\n",
       "  'continent',\n",
       "  'aided',\n",
       "  'remarkable',\n",
       "  'drill',\n",
       "  'devised',\n",
       "  'professor',\n",
       "  'frank',\n",
       "  'pabodie',\n",
       "  'engineering',\n",
       "  'department',\n",
       "  'wish',\n",
       "  'pioneer',\n",
       "  'other',\n",
       "  'field',\n",
       "  'did',\n",
       "  'hope',\n",
       "  'use',\n",
       "  'new',\n",
       "  'mechanical',\n",
       "  'different',\n",
       "  'points',\n",
       "  'along',\n",
       "  'previously',\n",
       "  'explored',\n",
       "  'paths',\n",
       "  'bring',\n",
       "  'light',\n",
       "  'materials',\n",
       "  'sort',\n",
       "  'hitherto',\n",
       "  'unreached',\n",
       "  'ordinary',\n",
       "  'methods',\n",
       "  'collection',\n",
       "  \"pabodie's\",\n",
       "  'drilling',\n",
       "  'apparatus',\n",
       "  'public',\n",
       "  'already',\n",
       "  'knows',\n",
       "  'reports',\n",
       "  'unique',\n",
       "  'radical',\n",
       "  'its',\n",
       "  'lightness',\n",
       "  'capacity',\n",
       "  'ordinary',\n",
       "  'drill',\n",
       "  'principle',\n",
       "  'principle',\n",
       "  'small',\n",
       "  'circular',\n",
       "  'rock',\n",
       "  'drill',\n",
       "  'such',\n",
       "  'way',\n",
       "  'quickly',\n",
       "  'strata',\n",
       "  'varying',\n",
       "  'hardness',\n",
       "  'steel',\n",
       "  'head',\n",
       "  'rods',\n",
       "  'gasoline',\n",
       "  'motor',\n",
       "  'wooden',\n",
       "  'derrick',\n",
       "  'dynamiting',\n",
       "  'paraphernalia',\n",
       "  'piping',\n",
       "  'bores',\n",
       "  'five',\n",
       "  'inches',\n",
       "  'wide',\n",
       "  'thousand',\n",
       "  'feet',\n",
       "  'deep',\n",
       "  'formed',\n",
       "  'needed',\n",
       "  'accessories',\n",
       "  'greater',\n",
       "  'load',\n",
       "  'three',\n",
       "  'sledges',\n",
       "  'carry',\n",
       "  'made',\n",
       "  'possible',\n",
       "  'clever',\n",
       "  'alloy',\n",
       "  'most',\n",
       "  'metal',\n",
       "  'objects',\n",
       "  'fashioned',\n",
       "  'four',\n",
       "  'large',\n",
       "  'aeroplanes',\n",
       "  'designed',\n",
       "  'especially',\n",
       "  'tremendous',\n",
       "  'altitude',\n",
       "  'flying',\n",
       "  'necessary',\n",
       "  'antarctic',\n",
       "  'plateau',\n",
       "  'added',\n",
       "  'devices',\n",
       "  'worked',\n",
       "  'pabodie',\n",
       "  'entire',\n",
       "  'expedition',\n",
       "  'base',\n",
       "  'edge',\n",
       "  'great',\n",
       "  'ice',\n",
       "  'barrier',\n",
       "  'various',\n",
       "  'suitable',\n",
       "  'inland',\n",
       "  'points',\n",
       "  'these',\n",
       "  'points',\n",
       "  'sufficient',\n",
       "  'quota',\n",
       "  'dogs',\n",
       "  'serve',\n",
       "  'planned',\n",
       "  'cover',\n",
       "  'great',\n",
       "  'area',\n",
       "  'antarctic',\n",
       "  'season',\n",
       "  'longer',\n",
       "  'absolutely',\n",
       "  'necessary',\n",
       "  'permit',\n",
       "  'operating',\n",
       "  'mostly',\n",
       "  'mountain',\n",
       "  'ranges',\n",
       "  'plateau',\n",
       "  'south',\n",
       "  'ross',\n",
       "  'sea',\n",
       "  'regions',\n",
       "  'explored',\n",
       "  'varying',\n",
       "  'degree',\n",
       "  'shackleton',\n",
       "  'amundsen',\n",
       "  'scott',\n",
       "  'byrd',\n",
       "  'frequent',\n",
       "  'changes',\n",
       "  'camp',\n",
       "  'made',\n",
       "  'aeroplane',\n",
       "  'involving',\n",
       "  'distances',\n",
       "  'great',\n",
       "  'enough',\n",
       "  'geological',\n",
       "  'significance',\n",
       "  'expected',\n",
       "  'unearth',\n",
       "  'quite',\n",
       "  'unprecedented',\n",
       "  'amount',\n",
       "  'material',\n",
       "  'especially',\n",
       "  'pre-cambrian',\n",
       "  'strata',\n",
       "  'narrow',\n",
       "  'range',\n",
       "  'antarctic',\n",
       "  'specimens',\n",
       "  'previously',\n",
       "  'secured',\n",
       "  'wished',\n",
       "  'also',\n",
       "  'obtain',\n",
       "  'great',\n",
       "  'possible',\n",
       "  'variety',\n",
       "  'upper',\n",
       "  'fossiliferous',\n",
       "  'rocks',\n",
       "  'since',\n",
       "  'primal',\n",
       "  'life',\n",
       "  'history',\n",
       "  'bleak',\n",
       "  'realm',\n",
       "  'ice',\n",
       "  'death',\n",
       "  'highest',\n",
       "  'importance',\n",
       "  'knowledge',\n",
       "  \"earth's\",\n",
       "  'past',\n",
       "  'antarctic',\n",
       "  'continent',\n",
       "  'once',\n",
       "  'even',\n",
       "  'tropical',\n",
       "  'vegetable',\n",
       "  'animal',\n",
       "  'life',\n",
       "  'lichens',\n",
       "  'marine',\n",
       "  'penguins',\n",
       "  'northern',\n",
       "  'edge',\n",
       "  'only',\n",
       "  'survivals',\n",
       "  'matter',\n",
       "  'common',\n",
       "  'information',\n",
       "  'hoped',\n",
       "  'expand',\n",
       "  'information',\n",
       "  'variety',\n",
       "  'accuracy',\n",
       "  'detail',\n",
       "  'simple',\n",
       "  'boring',\n",
       "  'revealed',\n",
       "  'fossiliferous',\n",
       "  'signs',\n",
       "  'aperture',\n",
       "  'blasting',\n",
       "  'order',\n",
       "  'get',\n",
       "  'specimens',\n",
       "  'suitable',\n",
       "  'size',\n",
       "  'condition',\n",
       "  'borings',\n",
       "  'varying',\n",
       "  'depth',\n",
       "  'according',\n",
       "  'promise',\n",
       "  'held',\n",
       "  'upper',\n",
       "  'soil',\n",
       "  'rock',\n",
       "  'confined',\n",
       "  'exposed',\n",
       "  'nearly',\n",
       "  'exposed',\n",
       "  'land',\n",
       "  'surfaces',\n",
       "  'these',\n",
       "  'inevitably',\n",
       "  'being',\n",
       "  'slopes',\n",
       "  'ridges',\n",
       "  'because',\n",
       "  'mile',\n",
       "  'two-mile',\n",
       "  'thickness',\n",
       "  'solid',\n",
       "  'ice',\n",
       "  'lower',\n",
       "  'levels',\n",
       "  'afford',\n",
       "  'waste',\n",
       "  'drilling',\n",
       "  'depth',\n",
       "  'considerable',\n",
       "  'amount',\n",
       "  'mere',\n",
       "  'glaciation',\n",
       "  'though',\n",
       "  'pabodie',\n",
       "  'worked',\n",
       "  'plan',\n",
       "  'sinking',\n",
       "  'copper',\n",
       "  'thick',\n",
       "  'clusters',\n",
       "  'borings',\n",
       "  'melting',\n",
       "  'off',\n",
       "  'limited',\n",
       "  'areas',\n",
       "  'ice',\n",
       "  'current',\n",
       "  'plan',\n",
       "  'put',\n",
       "  'effect',\n",
       "  'except',\n",
       "  'experimentally',\n",
       "  'expedition',\n",
       "  'such',\n",
       "  'ours',\n",
       "  'coming',\n",
       "  'starkweather-moore',\n",
       "  'expedition',\n",
       "  'follow',\n",
       "  'despite',\n",
       "  'warnings',\n",
       "  'issued',\n",
       "  'since',\n",
       "  'return',\n",
       "  'antarctic',\n",
       "  'public',\n",
       "  'knows',\n",
       "  'miskatonic',\n",
       "  'expedition',\n",
       "  'through',\n",
       "  'frequent',\n",
       "  'wireless',\n",
       "  'reports',\n",
       "  'arkham',\n",
       "  'associated',\n",
       "  'press',\n",
       "  'through',\n",
       "  'later',\n",
       "  'articles',\n",
       "  'pabodie',\n",
       "  'myself',\n",
       "  'consisted',\n",
       "  'four',\n",
       "  'men',\n",
       "  'university',\n",
       "  'pabodie',\n",
       "  'lake',\n",
       "  'biology',\n",
       "  'department',\n",
       "  'atwood',\n",
       "  'physics',\n",
       "  'department',\n",
       "  'also',\n",
       "  'myself',\n",
       "  'representing',\n",
       "  'geology',\n",
       "  'having',\n",
       "  'command',\n",
       "  'besides',\n",
       "  'sixteen',\n",
       "  'assistants',\n",
       "  'seven',\n",
       "  'graduate',\n",
       "  'students',\n",
       "  'miskatonic',\n",
       "  'nine',\n",
       "  'skilled',\n",
       "  'these',\n",
       "  'sixteen',\n",
       "  'twelve',\n",
       "  'aeroplane',\n",
       "  'pilots',\n",
       "  'two',\n",
       "  'whom',\n",
       "  'competent',\n",
       "  'wireless',\n",
       "  'eight',\n",
       "  'understood',\n",
       "  'navigation',\n",
       "  'compass',\n",
       "  'sextant',\n",
       "  'did',\n",
       "  'pabodie',\n",
       "  'atwood',\n",
       "  'addition',\n",
       "  'course',\n",
       "  'two',\n",
       "  'ships',\n",
       "  'wooden',\n",
       "  'reinforced',\n",
       "  'ice',\n",
       "  'conditions',\n",
       "  'having',\n",
       "  'auxiliary',\n",
       "  'steam',\n",
       "  'fully',\n",
       "  'foundation',\n",
       "  'aided',\n",
       "  'few',\n",
       "  'special',\n",
       "  'expedition',\n",
       "  'hence',\n",
       "  'preparations',\n",
       "  'extremely',\n",
       "  'thorough',\n",
       "  'despite',\n",
       "  'absence',\n",
       "  'great',\n",
       "  'publicity',\n",
       "  'dogs',\n",
       "  'sledges',\n",
       "  'camp',\n",
       "  'materials',\n",
       "  'parts',\n",
       "  'five',\n",
       "  'planes',\n",
       "  'delivered',\n",
       "  'boston',\n",
       "  'ships',\n",
       "  'loaded',\n",
       "  'marvelously',\n",
       "  'specific',\n",
       "  'purposes',\n",
       "  'matters',\n",
       "  'pertaining',\n",
       "  'supplies',\n",
       "  'transportation',\n",
       "  'camp',\n",
       "  'construction',\n",
       "  'excellent',\n",
       "  'example',\n",
       "  'many',\n",
       "  'recent',\n",
       "  'exceptionally',\n",
       "  'brilliant',\n",
       "  'predecessors',\n",
       "  'unusual',\n",
       "  'number',\n",
       "  'fame',\n",
       "  'these',\n",
       "  'predecessors',\n",
       "  'made',\n",
       "  'own',\n",
       "  'expedition',\n",
       "  'ample',\n",
       "  'though',\n",
       "  'noticed',\n",
       "  'world',\n",
       "  'large',\n",
       "  'newspapers',\n",
       "  'told',\n",
       "  'boston',\n",
       "  'harbor',\n",
       "  'september',\n",
       "  'nd',\n",
       "  'taking',\n",
       "  'leisurely',\n",
       "  'course',\n",
       "  'coast',\n",
       "  'through',\n",
       "  'panama',\n",
       "  'stopping',\n",
       "  'latter',\n",
       "  'place',\n",
       "  'took',\n",
       "  'final',\n",
       "  'supplies',\n",
       "  'none',\n",
       "  'exploring',\n",
       "  'party',\n",
       "  'ever',\n",
       "  'polar',\n",
       "  'regions',\n",
       "  'before',\n",
       "  'hence',\n",
       "  'relied',\n",
       "  'greatly',\n",
       "  'ship',\n",
       "  'captains',\n",
       "  'douglas',\n",
       "  'commanding',\n",
       "  'brig',\n",
       "  'arkham',\n",
       "  'serving',\n",
       "  'commander',\n",
       "  'sea',\n",
       "  'party',\n",
       "  'thorfinnssen',\n",
       "  'commanding',\n",
       "  'barque',\n",
       "  'miskatonic',\n",
       "  'both',\n",
       "  'antarctic',\n",
       "  'waters',\n",
       "  'left',\n",
       "  'inhabited',\n",
       "  'world',\n",
       "  'behind',\n",
       "  'sun',\n",
       "  'sank',\n",
       "  'lower',\n",
       "  'lower',\n",
       "  'north',\n",
       "  'stayed',\n",
       "  'longer',\n",
       "  'longer',\n",
       "  'above',\n",
       "  'horizon',\n",
       "  'each',\n",
       "  'day',\n",
       "  'south',\n",
       "  'latitude',\n",
       "  'first',\n",
       "  'table-like',\n",
       "  'objects',\n",
       "  'vertical',\n",
       "  'sides',\n",
       "  'just',\n",
       "  'before',\n",
       "  'reaching',\n",
       "  'antarctic',\n",
       "  'circle',\n",
       "  'crossed',\n",
       "  'october',\n",
       "  'th',\n",
       "  'quaint',\n",
       "  'considerably',\n",
       "  'troubled',\n",
       "  'field',\n",
       "  'ice',\n",
       "  'falling',\n",
       "  'temperature',\n",
       "  'bothered',\n",
       "  'considerably',\n",
       "  'after',\n",
       "  'long',\n",
       "  'voyage',\n",
       "  'through',\n",
       "  'tropics',\n",
       "  'tried',\n",
       "  'brace',\n",
       "  'worse',\n",
       "  'rigors',\n",
       "  'come',\n",
       "  'many',\n",
       "  'occasions',\n",
       "  'curious',\n",
       "  'atmospheric',\n",
       "  'effects',\n",
       "  'enchanted',\n",
       "  'vastly',\n",
       "  'these',\n",
       "  'including',\n",
       "  'vivid',\n",
       "  'mirage',\n",
       "  'first',\n",
       "  'ever',\n",
       "  'seen',\n",
       "  'distant',\n",
       "  'became',\n",
       "  'battlements',\n",
       "  'unimaginable',\n",
       "  'cosmic',\n",
       "  'castles',\n",
       "  'pushing',\n",
       "  'through',\n",
       "  'ice',\n",
       "  'fortunately',\n",
       "  'neither',\n",
       "  'extensive',\n",
       "  'nor',\n",
       "  'thickly',\n",
       "  'packed',\n",
       "  'regained',\n",
       "  'open',\n",
       "  'water',\n",
       "  'south',\n",
       "  'latitude',\n",
       "  'east',\n",
       "  'longitude',\n",
       "  'morning',\n",
       "  'october',\n",
       "  'th',\n",
       "  'strong',\n",
       "  'land',\n",
       "  'appeared',\n",
       "  'south',\n",
       "  'before',\n",
       "  'noon',\n",
       "  'felt',\n",
       "  'thrill',\n",
       "  'excitement',\n",
       "  'beholding',\n",
       "  'vast',\n",
       "  'lofty',\n",
       "  'mountain',\n",
       "  'chain',\n",
       "  'opened',\n",
       "  'covered',\n",
       "  'whole',\n",
       "  'vista',\n",
       "  'ahead',\n",
       "  'last',\n",
       "  'encountered',\n",
       "  'outpost',\n",
       "  'great',\n",
       "  'unknown',\n",
       "  'continent',\n",
       "  'its',\n",
       "  'cryptic',\n",
       "  'world',\n",
       "  'frozen',\n",
       "  'death',\n",
       "  'these',\n",
       "  'peaks',\n",
       "  'obviously',\n",
       "  'range',\n",
       "  'discovered',\n",
       "  'ross',\n",
       "  'task',\n",
       "  'round',\n",
       "  'cape',\n",
       "  'sail',\n",
       "  'east',\n",
       "  'coast',\n",
       "  'victoria',\n",
       "  'land',\n",
       "  'contemplated',\n",
       "  'base',\n",
       "  'shore',\n",
       "  'mcmurdo',\n",
       "  'sound',\n",
       "  'foot',\n",
       "  'volcano',\n",
       "  'erebus',\n",
       "  'south',\n",
       "  'latitude',\n",
       "  'last',\n",
       "  'lap',\n",
       "  'voyage',\n",
       "  'vivid',\n",
       "  'great',\n",
       "  'barren',\n",
       "  'peaks',\n",
       "  'mystery',\n",
       "  'loomed',\n",
       "  'constantly',\n",
       "  'against',\n",
       "  'west',\n",
       "  'low',\n",
       "  'northern',\n",
       "  'sun',\n",
       "  'noon',\n",
       "  'still',\n",
       "  'lower',\n",
       "  'southern',\n",
       "  'sun',\n",
       "  'midnight',\n",
       "  'poured',\n",
       "  'its',\n",
       "  'hazy',\n",
       "  'reddish',\n",
       "  'rays',\n",
       "  'white',\n",
       "  'snow',\n",
       "  'bluish',\n",
       "  'ice',\n",
       "  'water',\n",
       "  'lanes',\n",
       "  'black',\n",
       "  'bits',\n",
       "  'exposed',\n",
       "  'granite',\n",
       "  'slope',\n",
       "  'through',\n",
       "  'desolate',\n",
       "  'summits',\n",
       "  'swept',\n",
       "  'ranging',\n",
       "  'terrible',\n",
       "  'antarctic',\n",
       "  'wind',\n",
       "  'whose',\n",
       "  'sometimes',\n",
       "  'held',\n",
       "  'vague',\n",
       "  'suggestions',\n",
       "  'wild',\n",
       "  'musical',\n",
       "  'piping',\n",
       "  'notes',\n",
       "  'extending',\n",
       "  'wide',\n",
       "  'range',\n",
       "  'subconscious',\n",
       "  'reason',\n",
       "  'seemed',\n",
       "  'even',\n",
       "  'dimly',\n",
       "  'terrible',\n",
       "  'something',\n",
       "  'scene',\n",
       "  'reminded',\n",
       "  'strange',\n",
       "  'disturbing',\n",
       "  'asian',\n",
       "  'paintings',\n",
       "  'nicholas',\n",
       "  'roerich',\n",
       "  'still',\n",
       "  'stranger',\n",
       "  'disturbing',\n",
       "  'descriptions',\n",
       "  'evilly',\n",
       "  'fabled',\n",
       "  'plateau',\n",
       "  'leng',\n",
       "  'occur',\n",
       "  'dreaded',\n",
       "  'necronomicon',\n",
       "  'mad',\n",
       "  'arab',\n",
       "  'abdul',\n",
       "  'alhazred',\n",
       "  'rather',\n",
       "  'sorry',\n",
       "  'later',\n",
       "  'ever',\n",
       "  'looked',\n",
       "  'monstrous',\n",
       "  'book',\n",
       "  'college',\n",
       "  'library',\n",
       "  'th',\n",
       "  'november',\n",
       "  'sight',\n",
       "  'westward',\n",
       "  'range',\n",
       "  'having',\n",
       "  'temporarily',\n",
       "  'lost',\n",
       "  'passed',\n",
       "  'island',\n",
       "  'next',\n",
       "  'day',\n",
       "  'descried',\n",
       "  'cones',\n",
       "  'erebus',\n",
       "  'terror',\n",
       "  'ross',\n",
       "  'island',\n",
       "  'ahead',\n",
       "  'long',\n",
       "  'line',\n",
       "  'mountains',\n",
       "  'beyond',\n",
       "  'stretched',\n",
       "  'off',\n",
       "  'east',\n",
       "  'low',\n",
       "  'white',\n",
       "  'line',\n",
       "  'great',\n",
       "  'ice',\n",
       "  'barrier',\n",
       "  'rising',\n",
       "  'height',\n",
       "  'two',\n",
       "  'hundred',\n",
       "  'feet',\n",
       "  'rocky',\n",
       "  'cliffs',\n",
       "  'marking',\n",
       "  'end',\n",
       "  'southward',\n",
       "  'navigation',\n",
       "  'afternoon',\n",
       "  'entered',\n",
       "  'mcmurdo',\n",
       "  'sound',\n",
       "  'stood',\n",
       "  'off',\n",
       "  'coast',\n",
       "  'lee',\n",
       "  'smoking',\n",
       "  'mt',\n",
       "  'erebus',\n",
       "  'peak',\n",
       "  'towered',\n",
       "  'twelve',\n",
       "  'thousand',\n",
       "  'seven',\n",
       "  'hundred',\n",
       "  'feet',\n",
       "  'against',\n",
       "  'eastern',\n",
       "  'sky',\n",
       "  'print',\n",
       "  'sacred',\n",
       "  'while',\n",
       "  'beyond',\n",
       "  'rose',\n",
       "  'white',\n",
       "  'height',\n",
       "  'mt',\n",
       "  'terror',\n",
       "  'ten',\n",
       "  'thousand',\n",
       "  'nine',\n",
       "  'hundred',\n",
       "  'feet',\n",
       "  'altitude',\n",
       "  'extinct',\n",
       "  'volcano',\n",
       "  'puffs',\n",
       "  'smoke',\n",
       "  'erebus',\n",
       "  'came',\n",
       "  'graduate',\n",
       "  'assistants',\n",
       "  'brilliant',\n",
       "  'young',\n",
       "  'fellow',\n",
       "  'named',\n",
       "  'danforth',\n",
       "  'pointed',\n",
       "  'looked',\n",
       "  'snowy',\n",
       "  'slope',\n",
       "  'mountain',\n",
       "  'discovered',\n",
       "  'undoubtedly',\n",
       "  'source',\n",
       "  \"poe's\",\n",
       "  'image',\n",
       "  'wrote',\n",
       "  'seven',\n",
       "  'years',\n",
       "  'later',\n",
       "  'restlessly',\n",
       "  'roll',\n",
       "  'currents',\n",
       "  'yaanek',\n",
       "  'ultimate',\n",
       "  'pole',\n",
       "  'groan',\n",
       "  'roll',\n",
       "  'mount',\n",
       "  'yaanek',\n",
       "  'realms',\n",
       "  'pole',\n",
       "  'danforth',\n",
       "  ...]]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doc_tokens_cleaned[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def create_mallet_import(doc_tokens_cleaned, doc_labels, outpath = os.path.join('tutorial_supplementals', 'mallet_input')):\n",
    "    \"\"\"Creates files for mallet import.\n",
    "\n",
    "    Description:\n",
    "        With this function you can create preprocessed plain text files. \n",
    "        Commit a list of full paths or one single path as argument.\n",
    "        Use the function `remove_features_from_file()` to create a list of tokens\n",
    "        per document.\n",
    "\n",
    "    Args:\n",
    "        doc_tokens_cleaned Union(list[str], str): List of tokens per document\n",
    "        doc_labels list[str]: List of documents labels.\n",
    "\n",
    "    Todo:\n",
    "    \n",
    "    Example:\n",
    "        >>> doc_labels = ['examplefile']\n",
    "        >>> doc_tokens_cleaned = [['short', 'example', 'text']]\n",
    "        >>> create_mallet_import(doc_tokens_cleaned, doc_labels)\n",
    "        >>> outpath = os.path.join('tutorial_supplementals', 'mallet_input')\n",
    "        >>> os.path.isfile(os.path.join(outpath, 'examplefile.txt'))\n",
    "        True\n",
    "    \"\"\"\n",
    "    #log.info(\"Generating mallet input files ...\")\n",
    "    if not os.path.exists(outpath):\n",
    "                os.makedirs(outpath)\n",
    "    \n",
    "    for tokens, label in zip(doc_tokens_cleaned, doc_labels):\n",
    "        with open(os.path.join(outpath,label+'.txt'), 'w', encoding=\"utf-8\") as f:\n",
    "            f.write(str(tokens))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "create_mallet_import(doc_tokens_cleaned, doc_labels)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}