testing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\ifue3702\\AppData\\Local\\Continuum\\Anaconda3\\lib\\site-packages\\gensim\\utils.py:865: UserWarning: detected Windows; aliasing chunkize to chunkize_serial\n",
      "  warnings.warn(\"detected Windows; aliasing chunkize to chunkize_serial\")\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "MDS(dissimilarity='euclidean', eps=0.001, max_iter=300, metric=True,\n",
       "  n_components=2, n_init=4, n_jobs=1, random_state=None, verbose=0)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "import re\n",
    "import random\n",
    "import gensim\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import nltk\n",
    "from operator import setitem\n",
    "from six import iteritems\n",
    "from sklearn.cluster import KMeans\n",
    "from sklearn.cluster import MiniBatchKMeans\n",
    "from sklearn import metrics\n",
    "from sklearn.cluster import SpectralClustering\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem.snowball import SnowballStemmer\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from sklearn import cluster\n",
    "import h5py\n",
    "from sklearn.manifold import MDS\n",
    "MDS()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\ifue3702\\AppData\\Local\\Continuum\\Anaconda3\\lib\\site-packages\\fuzzywuzzy\\fuzz.py:35: UserWarning: Using slow pure-python SequenceMatcher. Install python-Levenshtein to remove this warning\n",
      "  warnings.warn('Using slow pure-python SequenceMatcher. Install python-Levenshtein to remove this warning')\n"
     ]
    },
    {
     "ename": "ImportError",
     "evalue": "No module named 'ParsingAndCleaningPapers2BuildDoc2VecModel'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mImportError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-07d1a5966841>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m from Litho.similarity import (check_similarity, match_lithcode, jaccard_similarity, \n\u001b[1;32m      3\u001b[0m                               calc_similarity_score, print_sim_compare, merge_similar_words)\n\u001b[0;32m----> 4\u001b[0;31m \u001b[1;32mfrom\u001b[0m \u001b[0mParsingAndCleaningPapers2BuildDoc2VecModel\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mclean\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlabeledSentences\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mImportError\u001b[0m: No module named 'ParsingAndCleaningPapers2BuildDoc2VecModel'"
     ]
    }
   ],
   "source": [
    "from Litho.nlp_funcs import *\n",
    "from Litho.similarity import (check_similarity, match_lithcode, jaccard_similarity, \n",
    "                              calc_similarity_score, print_sim_compare, merge_similar_words)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.manifold import TSNE\n",
    "from bokeh.plotting import figure, show, output_file\n",
    "from bokeh.models import ColumnDataSource, HoverTool\n",
    "\n",
    "\n",
    "def plot_embedding(arr, labels=None, filename='plot.html'):\n",
    "    \"\"\"Short summary.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    arr : np.array\n",
    "       2D array with embeddings.\n",
    "    labels : list(str)\n",
    "       List of labels (the default is None).\n",
    "    filename : str\n",
    "       Path to the output html file containing the plot (the default is 'plot.html').\n",
    "\n",
    "    \"\"\"\n",
    "    tsne = TSNE(n_components=2, verbose=0, perplexity=40, n_iter=300)\n",
    "    tsne_results = tsne.fit_transform(arr)\n",
    "\n",
    "    plot_data = {\n",
    "       'x': tsne_results[:, 0],\n",
    "       'y': tsne_results[:, 1]\n",
    "    }\n",
    "\n",
    "    plot_tools = ['pan', 'zoom_in', 'zoom_out']\n",
    "\n",
    "    if labels is not None:\n",
    "        plot_data['label'] = labels\n",
    "\n",
    "        hover = HoverTool(tooltips=[\n",
    "           (\"Description\", \"@label\")\n",
    "        ])\n",
    "\n",
    "        plot_tools.append(hover)\n",
    "\n",
    "    to_plot = ColumnDataSource(plot_data)\n",
    "    p = figure(title=\"Descriptions t-SNE\", tools=plot_tools)\n",
    "    p.xaxis.axis_label = ''\n",
    "    p.yaxis.axis_label = ''\n",
    "\n",
    "    p.circle('x', 'y', source=to_plot,\n",
    "            fill_alpha=0.8, size=10)\n",
    "\n",
    "    # p.add_layout(labels)\n",
    "    output_file(filename)\n",
    "    show(p)\n",
    "    \n",
    "# plot_embedding([doc2vec.infer_vector(x) for x in lemmatized[:1000]],\n",
    "#                [';'.join(x) for x in lemmatized[:1000]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def labeledSentences(tokenizedDescriptions, labels = None):\n",
    "    sentences = [gensim.models.doc2vec.TaggedDocument(words = i, tags = [labels[n]]) for n, i in enumerate(tokenizedDescriptions)]\n",
    "    return sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "root = os.getcwd()\n",
    "file = 'reclasified.csv'\n",
    "\n",
    "DF = pd.read_csv(os.path.join(root,file))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "stopa = nltk.corpus.stopwords.words('english')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "stopw2 = ['redish', 'reddish', 'red', 'black', 'blackish', 'brown', 'brownish',\n",
    "          'blue', 'blueish', 'orange', 'orangeish', 'gray', 'grey', 'grayish',\n",
    "          'greyish', 'white', 'whiteish', 'purple', 'purpleish', 'yellow',\n",
    "          'yellowish', 'green', 'greenish', 'light', 'very', 'pink', 'coloured', 'multicoloured',\n",
    "          'dark', 'color', 'colour', 'hard', 'soft', 'water', 'supply', 'fine', 'coarse',\n",
    "          'medium', 'bearing', 'pipe', 'sticky', 'tough', 'small', 'stiff', \n",
    "          'running', 'streaks', 'nominal', 'bands', 'back', 'slippery', 'loose', \n",
    "          'broken', 'fractured', 'surface', 'surface', 'rotten', 'compacted', 'seams',\n",
    "          'dry', 'wet', 'cemented', 'rock', 'trap', 'ridge', 'large', 'small', 'thin', 'soak',\n",
    "          'creek', 'ridge', 'clean', 'decomposed', 'band']\n",
    "\n",
    "\n",
    "stopa= stopa+stopw2  # add the additional stopwords above"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# stopw2 = ['redish', 'reddish', 'red', 'black', 'blackish', 'brown', 'brownish',\n",
    "#           'blue', 'blueish', 'orange', 'orangeish', 'gray', 'grey', 'grayish',\n",
    "#           'greyish', 'white', 'whiteish', 'purple', 'purpleish', 'yellow',\n",
    "#           'yellowish', 'green', 'greenish', 'light', 'very', 'pink', 'coloured', 'multicoloured',\n",
    "#           'dark', 'color', 'colour']\n",
    "# stopa= stopa+stopw2  # add the additional stopwords above"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## doc2vec model from descriptions (train-validation subsets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "msk = np.random.rand(len(DF)) < 0.8\n",
    "train = DF[msk]\n",
    "validation = DF[~msk]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "102167\n",
      "(102167,) (102167,)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[['sand'],\n",
       " ['clay'],\n",
       " ['clay'],\n",
       " ['clay', 'sandy'],\n",
       " ['drift'],\n",
       " ['sandstone'],\n",
       " ['sandstone'],\n",
       " ['clay', 'sandy'],\n",
       " ['clay'],\n",
       " ['clay', 'sandy'],\n",
       " ['sand', 'drift'],\n",
       " ['sand'],\n",
       " ['clay', 'sand'],\n",
       " ['clay', 'boulder'],\n",
       " ['sandstone'],\n",
       " ['clay'],\n",
       " ['clay', 'drift'],\n",
       " ['clay'],\n",
       " ['clay'],\n",
       " ['clay'],\n",
       " ['clay'],\n",
       " ['sand', 'drift'],\n",
       " ['clay'],\n",
       " ['clay'],\n",
       " ['drift'],\n",
       " ['clay'],\n",
       " ['drift', 'stoney'],\n",
       " ['drift'],\n",
       " ['cement'],\n",
       " ['clay', 'sand'],\n",
       " ['sand', 'drift'],\n",
       " ['clay'],\n",
       " ['limestone'],\n",
       " ['pebbles'],\n",
       " ['clay'],\n",
       " ['clay'],\n",
       " ['loam'],\n",
       " ['clay'],\n",
       " ['clay'],\n",
       " ['clay'],\n",
       " ['sandstone'],\n",
       " ['sand'],\n",
       " ['sandstone'],\n",
       " ['sandstone'],\n",
       " ['sand', 'silt'],\n",
       " ['silt'],\n",
       " ['shale'],\n",
       " ['sandstone'],\n",
       " ['sand'],\n",
       " ['sandstone'],\n",
       " ['clay'],\n",
       " ['sandstone', 'drift'],\n",
       " ['shale'],\n",
       " ['stones'],\n",
       " ['clay'],\n",
       " ['sandstone'],\n",
       " ['sand', 'drift'],\n",
       " ['sandstone'],\n",
       " ['loam', 'sandy'],\n",
       " ['clay'],\n",
       " ['sand', 'drift'],\n",
       " ['quartz'],\n",
       " ['clay'],\n",
       " ['clay'],\n",
       " ['clay'],\n",
       " ['clay'],\n",
       " ['sandstone'],\n",
       " ['clay'],\n",
       " ['sandstone'],\n",
       " ['sandstone'],\n",
       " ['sandstone'],\n",
       " ['sandstone'],\n",
       " ['sand'],\n",
       " ['sandstone'],\n",
       " ['clay', 'sandy'],\n",
       " ['ironstone', 'mixed'],\n",
       " ['clay'],\n",
       " ['sand'],\n",
       " ['clay'],\n",
       " ['clay', 'powdery'],\n",
       " ['sand'],\n",
       " ['sand'],\n",
       " ['sand'],\n",
       " ['loam', 'sandy'],\n",
       " ['gravel', 'sand'],\n",
       " ['sand'],\n",
       " ['clay'],\n",
       " ['clay'],\n",
       " ['clay'],\n",
       " ['clay', 'sandy'],\n",
       " ['sandstone'],\n",
       " ['clay', 'chalky'],\n",
       " ['clay', 'chalky'],\n",
       " ['clay', 'sandy'],\n",
       " ['clay'],\n",
       " ['gravel', 'sand'],\n",
       " ['clay'],\n",
       " ['sand'],\n",
       " ['clay'],\n",
       " ['gravel'],\n",
       " ['clay'],\n",
       " ['clay'],\n",
       " ['sand', 'drift'],\n",
       " ['clay'],\n",
       " ['gravel', 'stoney'],\n",
       " ['clay'],\n",
       " ['clay'],\n",
       " ['sand'],\n",
       " ['clay'],\n",
       " ['clay'],\n",
       " ['shale'],\n",
       " ['clay'],\n",
       " ['clay'],\n",
       " ['clay'],\n",
       " ['chalk'],\n",
       " ['clay'],\n",
       " ['clay'],\n",
       " ['sand'],\n",
       " ['sand'],\n",
       " ['sand'],\n",
       " ['sand'],\n",
       " ['sand'],\n",
       " ['shale'],\n",
       " ['clay'],\n",
       " ['sand'],\n",
       " ['sand'],\n",
       " ['sand'],\n",
       " ['clay'],\n",
       " ['clay'],\n",
       " ['sandstone'],\n",
       " ['clay'],\n",
       " ['sand'],\n",
       " ['clay'],\n",
       " ['sand'],\n",
       " ['shale'],\n",
       " ['sandstone'],\n",
       " ['sandstone'],\n",
       " ['sand'],\n",
       " ['shale'],\n",
       " ['sand'],\n",
       " ['clay'],\n",
       " ['shale', 'sandy'],\n",
       " ['shale'],\n",
       " ['sand'],\n",
       " ['shale'],\n",
       " ['shale'],\n",
       " ['clay', 'sand'],\n",
       " ['coal'],\n",
       " ['shale'],\n",
       " ['sandstone'],\n",
       " ['gravel', 'clay'],\n",
       " ['shale'],\n",
       " ['clay'],\n",
       " ['basalt'],\n",
       " ['shale'],\n",
       " ['clay', 'gravel'],\n",
       " ['shale'],\n",
       " [],\n",
       " ['sand'],\n",
       " ['sand'],\n",
       " ['clay'],\n",
       " ['shale'],\n",
       " ['shale'],\n",
       " ['clay'],\n",
       " ['diorite'],\n",
       " ['sand'],\n",
       " ['volcanic'],\n",
       " ['basalt'],\n",
       " ['clay'],\n",
       " ['shale'],\n",
       " ['loam', 'sandy'],\n",
       " ['sand'],\n",
       " ['sandstone'],\n",
       " ['sand'],\n",
       " ['clay', 'gritty'],\n",
       " ['clay'],\n",
       " ['drift'],\n",
       " ['sand'],\n",
       " ['sand', 'shale'],\n",
       " ['gravel', 'wash'],\n",
       " ['sandstone'],\n",
       " ['drift'],\n",
       " ['sandstone'],\n",
       " ['sandstone'],\n",
       " ['shale'],\n",
       " ['shale'],\n",
       " ['sand'],\n",
       " ['clay', 'drift'],\n",
       " ['sand'],\n",
       " ['clay'],\n",
       " ['sand'],\n",
       " ['sand', 'solid'],\n",
       " ['sandstone'],\n",
       " ['soil'],\n",
       " ['clay'],\n",
       " ['clay'],\n",
       " ['sand'],\n",
       " ['shale'],\n",
       " ['sand'],\n",
       " ['cement'],\n",
       " ['clay'],\n",
       " ['sandstone'],\n",
       " ['gravel'],\n",
       " ['sandstone'],\n",
       " ['sandstone'],\n",
       " ['clay'],\n",
       " ['clay'],\n",
       " ['sand'],\n",
       " ['sand'],\n",
       " ['clay'],\n",
       " ['clay'],\n",
       " ['shale'],\n",
       " ['sand'],\n",
       " ['sand'],\n",
       " ['clay'],\n",
       " ['clay'],\n",
       " ['clay'],\n",
       " ['sand'],\n",
       " ['sandstone'],\n",
       " ['ironstone'],\n",
       " ['gravel', 'drift'],\n",
       " ['clay'],\n",
       " ['clay', 'sand'],\n",
       " ['clay'],\n",
       " ['clay'],\n",
       " ['sand'],\n",
       " ['clay'],\n",
       " ['clay', 'sandy'],\n",
       " ['clay', 'sand'],\n",
       " ['shale'],\n",
       " ['sand', 'gravel'],\n",
       " ['sand'],\n",
       " ['clay'],\n",
       " ['sand'],\n",
       " ['clay'],\n",
       " ['sand'],\n",
       " ['sandstone'],\n",
       " ['clay', 'sandy'],\n",
       " ['sand'],\n",
       " ['soil'],\n",
       " ['clay'],\n",
       " ['conglomerate'],\n",
       " ['clay'],\n",
       " ['clay'],\n",
       " ['clay'],\n",
       " ['shale', 'sandy'],\n",
       " ['shale'],\n",
       " ['shale'],\n",
       " ['shale'],\n",
       " ['shale'],\n",
       " ['shale'],\n",
       " ['clay', 'sand'],\n",
       " ['sand', 'clay'],\n",
       " ['clay', 'puggy'],\n",
       " ['sand'],\n",
       " ['shale'],\n",
       " ['clay'],\n",
       " ['shale'],\n",
       " ['shale', 'cryptocrystalline'],\n",
       " ['clay', 'rubble'],\n",
       " ['clay', 'puggy', 'stones', 'rubble'],\n",
       " ['sand'],\n",
       " ['shale'],\n",
       " ['sandstone'],\n",
       " ['clay', 'sandy'],\n",
       " ['sandstone'],\n",
       " ['sandstone'],\n",
       " ['sandstone'],\n",
       " ['sand'],\n",
       " ['shale'],\n",
       " ['clay', 'sandy'],\n",
       " ['sand'],\n",
       " ['sand'],\n",
       " ['sand'],\n",
       " ['clay', 'sandy'],\n",
       " ['clay'],\n",
       " ['conglomerate'],\n",
       " ['sandstone'],\n",
       " ['sandstone'],\n",
       " ['shale'],\n",
       " ['sandstone'],\n",
       " ['soil'],\n",
       " ['coal'],\n",
       " ['coal'],\n",
       " ['shale'],\n",
       " ['basalt'],\n",
       " ['clay'],\n",
       " ['clay'],\n",
       " ['clay'],\n",
       " ['clay', 'sandy'],\n",
       " ['sand'],\n",
       " ['clay'],\n",
       " ['gravel'],\n",
       " ['sand'],\n",
       " ['loam', 'sandy'],\n",
       " ['clay'],\n",
       " ['sandstone'],\n",
       " ['clay'],\n",
       " ['sandstone'],\n",
       " ['sand'],\n",
       " ['clay'],\n",
       " ['gravel'],\n",
       " ['sand'],\n",
       " ['clay'],\n",
       " ['shale'],\n",
       " ['clay'],\n",
       " ['sand'],\n",
       " ['clay', 'sand'],\n",
       " ['clay', 'sand'],\n",
       " ['conglomerate'],\n",
       " ['shale', 'sandy'],\n",
       " ['clay'],\n",
       " ['clay'],\n",
       " ['clay', 'sand'],\n",
       " ['clay', 'gravel'],\n",
       " ['shale', 'sandy'],\n",
       " ['shale', 'sandy'],\n",
       " ['sandstone'],\n",
       " ['shale'],\n",
       " ['shale', 'sandy'],\n",
       " ['shale', 'sandy'],\n",
       " ['sand'],\n",
       " ['shale', 'sandy'],\n",
       " ['sandstone'],\n",
       " ['shale'],\n",
       " ['sandstone'],\n",
       " ['sand'],\n",
       " ['soil'],\n",
       " ['shale', 'sandy'],\n",
       " ['shale', 'sandy'],\n",
       " ['clay'],\n",
       " ['sandstone'],\n",
       " ['sandstone'],\n",
       " ['shale', 'sandy'],\n",
       " ['sand'],\n",
       " ['sandstone'],\n",
       " ['sandstone'],\n",
       " ['sandstone'],\n",
       " ['shale', 'sandy'],\n",
       " ['shale', 'sandy'],\n",
       " ['sand'],\n",
       " ['sand'],\n",
       " ['clay', 'sandy'],\n",
       " ['shale'],\n",
       " ['shale'],\n",
       " ['shale'],\n",
       " ['sandstone'],\n",
       " ['shale'],\n",
       " ['shale', 'sandy'],\n",
       " ['sandstone'],\n",
       " ['sandstone'],\n",
       " ['shale'],\n",
       " ['shale'],\n",
       " ['sandstone'],\n",
       " ['basalt'],\n",
       " ['sandstone'],\n",
       " ['clay'],\n",
       " ['basalt'],\n",
       " ['basalt'],\n",
       " ['basalt'],\n",
       " ['basalt'],\n",
       " ['boulders'],\n",
       " ['drift'],\n",
       " ['sand'],\n",
       " ['sand'],\n",
       " ['shale'],\n",
       " ['sand'],\n",
       " ['sand'],\n",
       " ['sand'],\n",
       " ['sand'],\n",
       " ['sand'],\n",
       " ['gravel'],\n",
       " ['clay', 'sandstone'],\n",
       " ['clay', 'sandy'],\n",
       " ['sandstone'],\n",
       " ['clay'],\n",
       " ['clay'],\n",
       " ['sandstone'],\n",
       " ['clay', 'gravel'],\n",
       " ['clay'],\n",
       " ['clay'],\n",
       " ['quartz', 'flinty'],\n",
       " ['clay'],\n",
       " ['clay'],\n",
       " ['sandstone', 'clay', 'interlayere'],\n",
       " ['sand'],\n",
       " ['pebbles'],\n",
       " ['sandstone'],\n",
       " ['soil'],\n",
       " ['clay'],\n",
       " ['sandstone'],\n",
       " ['sandstone'],\n",
       " ['quartz'],\n",
       " ['clay'],\n",
       " ['shale'],\n",
       " ['sandstone'],\n",
       " ['clay'],\n",
       " ['coal', 'formation', 'strata'],\n",
       " ['sandstone'],\n",
       " ['shale'],\n",
       " ['ironstone'],\n",
       " ['sandstone'],\n",
       " ['sandstone'],\n",
       " ['clay'],\n",
       " ['sandstone'],\n",
       " ['clay'],\n",
       " ['soil'],\n",
       " ['sandstone'],\n",
       " ['shale'],\n",
       " ['sandstone'],\n",
       " ['clay', 'pebbles', 'pebbly'],\n",
       " ['clay', 'sandy'],\n",
       " ['conglomerate'],\n",
       " ['clay'],\n",
       " ['sand'],\n",
       " ['sand'],\n",
       " ['clay', 'sandy'],\n",
       " ['clay'],\n",
       " ['clay'],\n",
       " ['clay', 'sandy'],\n",
       " ['sandstone'],\n",
       " ['sandstone'],\n",
       " ['clay'],\n",
       " ['conglomerate'],\n",
       " ['clay', 'sandy'],\n",
       " ['sand'],\n",
       " ['sandstone'],\n",
       " ['clay', 'sandy'],\n",
       " ['gravel'],\n",
       " ['sandstone'],\n",
       " ['shale', 'sandy'],\n",
       " ['clay'],\n",
       " ['shale', 'sandy'],\n",
       " ['clay', 'sandy'],\n",
       " ['shale'],\n",
       " ['clay'],\n",
       " ['sandstone', 'sand'],\n",
       " ['shale'],\n",
       " ['shale', 'sandy'],\n",
       " ['shale'],\n",
       " ['sand'],\n",
       " ['clay'],\n",
       " ['sandstone'],\n",
       " ['clay'],\n",
       " ['sandstone'],\n",
       " ['clay'],\n",
       " ['clay'],\n",
       " ['sandstone'],\n",
       " ['clay'],\n",
       " ['clay', 'sandstone'],\n",
       " ['shale', 'sandy'],\n",
       " ['shale', 'sandy'],\n",
       " ['sandstone'],\n",
       " ['sandstone', 'shale'],\n",
       " ['shale', 'puggy'],\n",
       " ['shale'],\n",
       " ['shale'],\n",
       " ['clay'],\n",
       " ['shale'],\n",
       " ['sand'],\n",
       " ['sandstone'],\n",
       " ['sandstone'],\n",
       " ['shale'],\n",
       " ['shale', 'sandy'],\n",
       " ['shale'],\n",
       " ['shale'],\n",
       " ['stones'],\n",
       " ['sand'],\n",
       " ['shale'],\n",
       " ['clay'],\n",
       " ['soil'],\n",
       " ['clay'],\n",
       " ['clay'],\n",
       " ['shale'],\n",
       " ['shale', 'sandy'],\n",
       " ['shale', 'sandy'],\n",
       " ['clay'],\n",
       " ['sandstone'],\n",
       " ['sandstone', 'clay'],\n",
       " ['clay', 'sandy'],\n",
       " ['sandstone'],\n",
       " ['clay'],\n",
       " ['shale', 'sandy'],\n",
       " ['clay'],\n",
       " ['clay'],\n",
       " ['shale', 'sandy'],\n",
       " ['fireclay'],\n",
       " ['sandstone'],\n",
       " ['clay'],\n",
       " ['clay'],\n",
       " ['shale'],\n",
       " ['clay'],\n",
       " ['clay'],\n",
       " ['shale'],\n",
       " ['shale'],\n",
       " ['shale'],\n",
       " ['sand'],\n",
       " ['clay'],\n",
       " ['shale'],\n",
       " ['shale'],\n",
       " ['sand', 'gravel'],\n",
       " ['shale'],\n",
       " ['shale'],\n",
       " ['shale'],\n",
       " ['shale'],\n",
       " ['clay'],\n",
       " ['sandstone'],\n",
       " ['sandstone'],\n",
       " ['clay', 'gravel'],\n",
       " ['clay', 'gritty'],\n",
       " ['clay', 'gritty'],\n",
       " ['shale'],\n",
       " ['sandstone'],\n",
       " ['clay', 'gritty'],\n",
       " ['clay', 'gritty'],\n",
       " ['clay'],\n",
       " ['clay'],\n",
       " ['sandstone'],\n",
       " ['clay'],\n",
       " ['shale', 'sandy'],\n",
       " ['sand'],\n",
       " ['shale'],\n",
       " ['clay', 'gritty'],\n",
       " ['shale', 'coal'],\n",
       " ['shale'],\n",
       " ['subsoil'],\n",
       " ['clay', 'stones'],\n",
       " ['rubble', 'drift'],\n",
       " ['clay'],\n",
       " ['sand'],\n",
       " ['shale', 'sandy'],\n",
       " ['sandstone'],\n",
       " ['clay'],\n",
       " ['shale', 'sandy'],\n",
       " ['shale'],\n",
       " ['shale', 'sandy'],\n",
       " ['sandstone'],\n",
       " ['sandstone'],\n",
       " ['clay'],\n",
       " ['sand'],\n",
       " ['shale'],\n",
       " ['shale', 'sandy'],\n",
       " ['clay'],\n",
       " ['shale'],\n",
       " ['clay'],\n",
       " ['clay'],\n",
       " ['shale', 'sandy'],\n",
       " ['sandstone'],\n",
       " ['shale'],\n",
       " ['sand'],\n",
       " ['soil'],\n",
       " ['shale'],\n",
       " ['shale'],\n",
       " ['clay'],\n",
       " ['sandstone'],\n",
       " ['shale'],\n",
       " ['shale'],\n",
       " ['sandstone'],\n",
       " ['clay', 'gritty'],\n",
       " ['sandstone'],\n",
       " ['shale'],\n",
       " ['clay'],\n",
       " ['clay'],\n",
       " ['shale'],\n",
       " ['sandstone'],\n",
       " ['sand'],\n",
       " ['shale'],\n",
       " ['shale'],\n",
       " ['shale'],\n",
       " ['shale', 'sandy'],\n",
       " ['shale', 'sandy'],\n",
       " ['clay'],\n",
       " ['shale', 'sandy', 'sand'],\n",
       " ['shale'],\n",
       " ['shale', 'sandy', 'sand'],\n",
       " ['shale'],\n",
       " ['clay'],\n",
       " ['sandstone'],\n",
       " ['clay'],\n",
       " ['sandstone'],\n",
       " ['sandstone'],\n",
       " ['sandstone'],\n",
       " ['clay', 'gritty'],\n",
       " ['sandstone'],\n",
       " ['sandstone'],\n",
       " ['sand'],\n",
       " ['sandstone'],\n",
       " ['clay'],\n",
       " ['sandstone'],\n",
       " ['sandstone'],\n",
       " ['shale'],\n",
       " ['clay'],\n",
       " ['shale'],\n",
       " ['sandstone'],\n",
       " ['shale', 'sandy'],\n",
       " ['sandstone'],\n",
       " ['clay'],\n",
       " ['shale'],\n",
       " ['drift'],\n",
       " ['sandstone'],\n",
       " ['shale'],\n",
       " ['shale'],\n",
       " ['drift'],\n",
       " ['sandstone'],\n",
       " ['sandstone'],\n",
       " ['clay'],\n",
       " ['sandstone'],\n",
       " ['sandstone'],\n",
       " ['sandstone'],\n",
       " ['clay', 'gritty'],\n",
       " ['clay', 'stoney'],\n",
       " ['sand'],\n",
       " ['clay', 'gritty'],\n",
       " ['coal', 'shale'],\n",
       " ['clay', 'gravel'],\n",
       " ['sand'],\n",
       " ['clay', 'gravel'],\n",
       " ['sandstone'],\n",
       " ['shale'],\n",
       " ['clay', 'gritty'],\n",
       " ['sand'],\n",
       " ['shale'],\n",
       " [],\n",
       " ['sandstone'],\n",
       " ['clay'],\n",
       " ['clay'],\n",
       " ['clay'],\n",
       " ['clay'],\n",
       " ['sand', 'gravel'],\n",
       " ['sandstone'],\n",
       " ['sand', 'drift'],\n",
       " ['clay'],\n",
       " ['sandstone'],\n",
       " ['clay'],\n",
       " ['gravel'],\n",
       " ['sandstone'],\n",
       " ['sandstone'],\n",
       " ['clay', 'rising'],\n",
       " ['sandstone'],\n",
       " ['shale'],\n",
       " ['sand'],\n",
       " ['shale'],\n",
       " ['sandstone', 'sand'],\n",
       " ['clay'],\n",
       " ['clay'],\n",
       " ['shale'],\n",
       " ['shale'],\n",
       " ['shale'],\n",
       " ['clay', 'sandy'],\n",
       " ['sandstone'],\n",
       " ['shale'],\n",
       " ['shale'],\n",
       " ['clay'],\n",
       " ['sandstone'],\n",
       " ['sand'],\n",
       " ['clay', 'gritty'],\n",
       " ['shale'],\n",
       " ['sandstone'],\n",
       " ['clay', 'gritty'],\n",
       " ['clay'],\n",
       " ['gravel', 'sand'],\n",
       " ['clay', 'gritty'],\n",
       " ['sandstone'],\n",
       " ['sand', 'boulder'],\n",
       " ['shale'],\n",
       " ['sand', 'boulder'],\n",
       " ['shale'],\n",
       " ['conglomerate'],\n",
       " ['clay'],\n",
       " ['sandstone'],\n",
       " ['boulders'],\n",
       " ['shale'],\n",
       " ['sandstone'],\n",
       " ['shale'],\n",
       " ['clay'],\n",
       " ['clay'],\n",
       " ['shale'],\n",
       " ['sandstone'],\n",
       " ['shale'],\n",
       " ['clay'],\n",
       " ['sandstone'],\n",
       " ['slate'],\n",
       " ['sand', 'gravel'],\n",
       " ['clay', 'gritty'],\n",
       " ['clay'],\n",
       " ['sand'],\n",
       " ['clay', 'gritty'],\n",
       " ['sandstone'],\n",
       " ['clay', 'sandy'],\n",
       " ['sandstone'],\n",
       " ['shale', 'sandy'],\n",
       " ['clay'],\n",
       " ['sandstone'],\n",
       " ['shale', 'sandy'],\n",
       " ['shale', 'puggy'],\n",
       " ['sandstone'],\n",
       " ['shale'],\n",
       " ['sandstone'],\n",
       " ['sandstone'],\n",
       " ['sandstone'],\n",
       " ['soil'],\n",
       " ['clay'],\n",
       " ['gravel'],\n",
       " ['clay'],\n",
       " ['clay', 'gravel'],\n",
       " ['clay'],\n",
       " ['sandstone'],\n",
       " ['cement'],\n",
       " ['sandstone'],\n",
       " ['clay', 'gritty'],\n",
       " ['gravel', 'clay'],\n",
       " ['clay'],\n",
       " ['clay'],\n",
       " ['clay', 'sandy'],\n",
       " ['shale', 'sandy'],\n",
       " ['shale'],\n",
       " ['shale'],\n",
       " ['clay'],\n",
       " ['clay'],\n",
       " ['drift'],\n",
       " ['sandstone'],\n",
       " ['sandstone', 'shale'],\n",
       " ['boulders'],\n",
       " ['sandstone'],\n",
       " ['shale', 'puggy'],\n",
       " ['sandstone'],\n",
       " ['sandstone'],\n",
       " ['shale', 'sandy'],\n",
       " ['sandstone'],\n",
       " ['shale', 'sandstone'],\n",
       " ['clay', 'gritty'],\n",
       " ['shale'],\n",
       " ['shale'],\n",
       " ['clay', 'gritty'],\n",
       " ['sandstone'],\n",
       " ['sandstone'],\n",
       " ['clay'],\n",
       " ['clay', 'gritty'],\n",
       " ['shale'],\n",
       " ['sand', 'gravel'],\n",
       " ['sandstone'],\n",
       " ['clay'],\n",
       " ['gravel'],\n",
       " ['clay'],\n",
       " ['clay'],\n",
       " ['clay', 'gritty'],\n",
       " ['clay', 'gritty'],\n",
       " ['sand', 'gravel', 'gritty', 'clay'],\n",
       " ['soil', 'clay'],\n",
       " ['clay', 'layers'],\n",
       " ['formation', 'strata', 'sandy', 'clay'],\n",
       " ['clay', 'gritty'],\n",
       " ['shale'],\n",
       " ['sandstone'],\n",
       " ['sand'],\n",
       " ['shale'],\n",
       " ['sand'],\n",
       " ['shale'],\n",
       " ['shale'],\n",
       " ['soil'],\n",
       " ['sandstone'],\n",
       " ['clay'],\n",
       " ['sand'],\n",
       " ['shale'],\n",
       " ['shale'],\n",
       " ['shale'],\n",
       " ['shale'],\n",
       " ['shale', 'mixed'],\n",
       " ['shale', 'mixed'],\n",
       " ['shale', 'mixed'],\n",
       " ['clay'],\n",
       " ['basalt'],\n",
       " ['sand'],\n",
       " ['quartz'],\n",
       " ['shale'],\n",
       " ['basalt'],\n",
       " ['clay'],\n",
       " ['sandstone'],\n",
       " ['sand', 'gravel'],\n",
       " ['clay', 'gravel'],\n",
       " ['clay'],\n",
       " ['sandstone'],\n",
       " ['clay', 'sandy'],\n",
       " ['soil'],\n",
       " ['basalt'],\n",
       " ['clay'],\n",
       " ['basalt'],\n",
       " ['basalt'],\n",
       " ['basalt'],\n",
       " ['basalt'],\n",
       " ['basalt'],\n",
       " ['basalt'],\n",
       " ['basalt'],\n",
       " ['clay'],\n",
       " ['sand', 'gravel'],\n",
       " ['quartz', 'gravel'],\n",
       " ['clay'],\n",
       " ['sandstone'],\n",
       " ['shale'],\n",
       " ['clay', 'gravel'],\n",
       " ['shale', 'sandy'],\n",
       " ['shale'],\n",
       " ['shale'],\n",
       " ['clay'],\n",
       " ['clay', 'gritty'],\n",
       " ['boulders', 'basalt'],\n",
       " ['clay'],\n",
       " ['basalt'],\n",
       " ['shale'],\n",
       " ['clay'],\n",
       " ['shale'],\n",
       " ['shale'],\n",
       " ['shale'],\n",
       " ['sandstone', 'shale'],\n",
       " ['sandstone'],\n",
       " ['shale'],\n",
       " ['sandstone'],\n",
       " ['shale'],\n",
       " ['gravel'],\n",
       " ['shale'],\n",
       " ['sandstone'],\n",
       " ['gravel'],\n",
       " ['shale'],\n",
       " ['shale'],\n",
       " ['shale', 'sandstone'],\n",
       " ['clay'],\n",
       " ['sandstone'],\n",
       " ['shale'],\n",
       " ['sandstone', 'gravel'],\n",
       " ['shale', 'sandstone'],\n",
       " ['sandstone', 'shale'],\n",
       " ['sandstone'],\n",
       " ['sandstone'],\n",
       " ['shale'],\n",
       " ['shale'],\n",
       " ['limestone'],\n",
       " ['sandstone'],\n",
       " ['shale'],\n",
       " ['shale'],\n",
       " ['sandstone'],\n",
       " ['sand'],\n",
       " ['sandstone'],\n",
       " ['shale'],\n",
       " ['shale', 'sandstone'],\n",
       " ['sandstone'],\n",
       " ['sandstone'],\n",
       " ['clay'],\n",
       " ['sandstone'],\n",
       " ['clay'],\n",
       " ['sand', 'gravel'],\n",
       " ['clay', 'swelling'],\n",
       " ['sand', 'clay'],\n",
       " ['clay', 'sandy'],\n",
       " ['drift'],\n",
       " ['clay'],\n",
       " ['clay'],\n",
       " ['shale'],\n",
       " ['shale'],\n",
       " ['clay', 'sandy'],\n",
       " ['clay'],\n",
       " ['clay', 'sandy'],\n",
       " ['clay'],\n",
       " ['clay'],\n",
       " ['sandstone'],\n",
       " ['clay'],\n",
       " ['sandstone'],\n",
       " ['sandstone'],\n",
       " ['clay'],\n",
       " ['clay', 'gravel'],\n",
       " ['shale'],\n",
       " ['shale', 'clay'],\n",
       " ['sandstone'],\n",
       " ['clay'],\n",
       " ['clay', 'sandy'],\n",
       " ['shale'],\n",
       " ['shale'],\n",
       " ['clay', 'sand'],\n",
       " ['shale'],\n",
       " ['clay', 'sandy', 'stones'],\n",
       " ['sand', 'gravel'],\n",
       " ['clay', 'sandy'],\n",
       " ['clay', 'sandy'],\n",
       " ['clay', 'sandy'],\n",
       " ['sandstone'],\n",
       " ['shale', 'sandy'],\n",
       " ['shale'],\n",
       " ['sand', 'gravel'],\n",
       " ['clay'],\n",
       " ['shale'],\n",
       " ['shale'],\n",
       " ['sand'],\n",
       " ['shale'],\n",
       " ['sandstone'],\n",
       " ['sand'],\n",
       " ['shale'],\n",
       " ['shale', 'sandy'],\n",
       " ['coal'],\n",
       " ['shale'],\n",
       " ['sandstone'],\n",
       " ['sandstone'],\n",
       " ['sandstone'],\n",
       " ['sandstone'],\n",
       " ['shale'],\n",
       " ['sandstone'],\n",
       " ['sand'],\n",
       " ['shale'],\n",
       " ['sandstone'],\n",
       " ['sandstone'],\n",
       " ['shale'],\n",
       " ['shale'],\n",
       " ['sandstone'],\n",
       " ['shale', 'sandy'],\n",
       " ['shale'],\n",
       " ['shale'],\n",
       " ['sandstone'],\n",
       " ['shale', 'unknown'],\n",
       " ['sand', 'unknown'],\n",
       " ['shale'],\n",
       " ['shale'],\n",
       " ['sandstone'],\n",
       " ['sandstone'],\n",
       " ['shale'],\n",
       " ['clay', 'sandy'],\n",
       " ['clay', 'sandy'],\n",
       " ['clay'],\n",
       " ['clay', 'sandy'],\n",
       " ['clay', 'sandy'],\n",
       " ['clay', 'sandy'],\n",
       " ['clay', 'sandy'],\n",
       " ['sandstone'],\n",
       " ['clay', 'sandstone'],\n",
       " ['sand', 'clay'],\n",
       " ['rubble', 'heavy'],\n",
       " ['clay', 'sandy'],\n",
       " ['shale'],\n",
       " ['clay'],\n",
       " ['sandstone'],\n",
       " ['sandstone'],\n",
       " ['diorite'],\n",
       " ['sandstone'],\n",
       " ['clay', 'boulder'],\n",
       " ['sandstone'],\n",
       " ['shale'],\n",
       " ['shale', 'sandstone'],\n",
       " ['sandstone', 'clay', 'boulder'],\n",
       " ['shale', 'boulder'],\n",
       " ['shale', 'sandstone'],\n",
       " ['shale'],\n",
       " ['sandstone'],\n",
       " ['clay', 'gravel'],\n",
       " ['boulders', 'gravel'],\n",
       " ['clay'],\n",
       " ['sand'],\n",
       " ['shale', 'clay'],\n",
       " ['sandstone'],\n",
       " ['sandstone'],\n",
       " ['clay'],\n",
       " ['sandstone'],\n",
       " ['clay', 'shale'],\n",
       " ['clay', 'sandstone'],\n",
       " ['sandstone'],\n",
       " ['sandstone', 'shale'],\n",
       " ['clay'],\n",
       " ['shale'],\n",
       " ['sandstone'],\n",
       " ['shale'],\n",
       " ['sandstone'],\n",
       " ['shale'],\n",
       " ['sandstone'],\n",
       " ['sandstone'],\n",
       " ['sandstone'],\n",
       " ['clay'],\n",
       " ['shale'],\n",
       " ['clay'],\n",
       " ['shale'],\n",
       " ['shale'],\n",
       " ['sandstone'],\n",
       " ['sandstone'],\n",
       " ['shale'],\n",
       " ['coal'],\n",
       " ['sandstone'],\n",
       " ['shale'],\n",
       " ['shale', 'sandstone'],\n",
       " ['sandstone'],\n",
       " ['sandstone'],\n",
       " ['shale'],\n",
       " ['shale'],\n",
       " ['sandstone'],\n",
       " ['sand'],\n",
       " ['shale'],\n",
       " ['sandstone'],\n",
       " ['sandstone', 'shale'],\n",
       " ['shale'],\n",
       " ['sandstone'],\n",
       " ['clay', 'cement'],\n",
       " ['sandstone'],\n",
       " ['sandstone', 'shale'],\n",
       " ['sandstone'],\n",
       " ['shale'],\n",
       " ['sandy'],\n",
       " ...]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = train.dropna(how = 'any')\n",
    "Tags = train['OWN'].tolist()\n",
    "print(len(Tags))\n",
    "Descriptions=train.Description.tolist()\n",
    "Descriptions = [str(n) for n in Descriptions]\n",
    "\n",
    "#use extend so it's a big flat list of vocab\n",
    "totalvocab_lemmatized = []\n",
    "totalvocab_tokenized = []\n",
    "for i in Descriptions:\n",
    "    allwords_stemmed = tokenize_and_lemmatize(i, stopa) #for each item in 'synopses', tokenize/stem\n",
    "    totalvocab_lemmatized.extend([allwords_stemmed]) #extend the 'totalvocab_stemmed' list\n",
    "\n",
    "    allwords_tokenized = tokenize_only(i, stopa)\n",
    "    totalvocab_tokenized.extend([allwords_tokenized])\n",
    "\n",
    "print(np.shape(totalvocab_tokenized),  np.shape(totalvocab_lemmatized))\n",
    "Sentences = totalvocab_tokenized\n",
    "Sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "labSen =  labeledSentences(Sentences, labels = Tags)\n",
    "model = gensim.models.Doc2Vec(size = 200,\n",
    "                             min_count = 1,\n",
    "                             workers = 8,\n",
    "                             iter = 100)\n",
    "model.build_vocab(labSen)\n",
    "#training of model\n",
    "for epoch in range(20):\n",
    "    print('iteration '+str(epoch+1))\n",
    "    model.train(labSen, total_examples=len(labSen), epochs = 20)\n",
    "    model.alpha -= 0.002\n",
    "    model.min_alpha = model.alpha"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "validation = validation.dropna(how = 'any')\n",
    "# classes = validation['OWN'].tolist()\n",
    "\n",
    "text=validation.Description.tolist()\n",
    "text = [str(n) for n in text]\n",
    "\n",
    "#use extend so it's a big flat list of vocab\n",
    "lemmatized = []\n",
    "tokenized = []\n",
    "for i in text:\n",
    "    allwords_stemmed = tokenize_and_lemmatize(i, stopa) #for each item in 'synopses', tokenize/stem\n",
    "    lemmatized.extend([allwords_stemmed]) #extend the 'totalvocab_stemmed' list\n",
    "\n",
    "    allwords_tokenized = tokenize_only(i, stopa)\n",
    "    tokenized.extend([allwords_tokenized])\n",
    "\n",
    "print(np.shape(tokenized),  np.shape(lemmatized))\n",
    "texts = tokenized"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "vectors = []\n",
    "for n in tokenized:\n",
    "    vectors.append(np.reshape(model.infer_vector(n), (1,200)))\n",
    "validation['vectors'] = vectors\n",
    "\n",
    "X = np.vstack(validation.vectors)\n",
    "kmeans = cluster.MiniBatchKMeans(n_clusters = len(validation.OWN.unique().tolist())).fit(X)\n",
    "pred_classes = kmeans.predict(X)\n",
    "print(metrics.adjusted_rand_score( validation['OWN'].tolist(), pred_classes.tolist()),\n",
    "      metrics.adjusted_mutual_info_score( validation['OWN'].tolist(), pred_classes.tolist()), \n",
    "      metrics.completeness_score( validation['OWN'].tolist(), pred_classes.tolist()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dictionary building from descriptions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(127930,) (127930,)\n"
     ]
    }
   ],
   "source": [
    "DF = DF[['Description']]\n",
    "DF = DF.dropna(how = 'any')\n",
    "Descriptions=DF.Description.tolist()\n",
    "Descriptions = [str(n) for n in Descriptions]\n",
    "\n",
    "#use extend so it's a big flat list of vocab\n",
    "totalvocab_lemmatized = []\n",
    "totalvocab_tokenized = []\n",
    "for i in Descriptions:\n",
    "    allwords_stemmed = tokenize_and_lemmatize(i, stopa) #for each item in 'synopses', tokenize/stem\n",
    "    totalvocab_lemmatized.extend([allwords_stemmed]) #extend the 'totalvocab_stemmed' list\n",
    "\n",
    "    allwords_tokenized = tokenize_only(i, stopa)\n",
    "    totalvocab_tokenized.extend([allwords_tokenized])\n",
    "\n",
    "print(np.shape(totalvocab_tokenized),  np.shape(totalvocab_lemmatized))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dictionary(1205 unique tokens: ['gyre', 'cobble', 'lava', 'drilling', 'limit']...)\n"
     ]
    }
   ],
   "source": [
    "###here just tokens or stemmed tokens can be used alternatively\n",
    "dictionary = gensim.corpora.Dictionary(totalvocab_lemmatized)\n",
    "## newt two lines remove tokens contained just once in the dictionary\n",
    "once_ids = [tokenid for tokenid, docfreq in iteritems(dictionary.dfs) if docfreq == 1]\n",
    "dictionary.filter_tokens(once_ids)\n",
    "\n",
    "dictionary.compactify()\n",
    "print(dictionary)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(127930,) (127930,)\n",
      "length corpus 127626\n"
     ]
    }
   ],
   "source": [
    "#use extend so it's a big flat list of vocab\n",
    "Namoi = 'reclasified.csv'\n",
    "Data = pd.read_csv(os.path.join(root,Namoi))\n",
    "Data = Data.dropna(subset=['Description'])\n",
    "Descriptions = Data[['Description']]\n",
    "Descriptions=Descriptions.Description.tolist()\n",
    "Descriptions = [str(n) for n in Descriptions]\n",
    "lemm_total = []\n",
    "token_total = []\n",
    "for i in Descriptions:\n",
    "    allwords_stemmed = tokenize_and_lemmatize(i, stopa) #for each item in 'synopses', tokenize/stem\n",
    "    lemm_total.extend([allwords_stemmed]) #extend the 'totalvocab_stemmed' list\n",
    "\n",
    "    allwords_tokenized = tokenize_only(i, stopa)\n",
    "    token_total.extend([allwords_tokenized])\n",
    "\n",
    "print(np.shape(token_total),  np.shape(lemm_total))\n",
    "Data['words'] = [len(n) for n in token_total]\n",
    "Data['Tokens'] = token_total\n",
    "Data['Lemmas'] = lemm_total\n",
    "Data = Data[Data.words > 0]\n",
    "corpus = [dictionary.doc2bow(text) for text in Data.Tokens.tolist()]\n",
    "print('length corpus', len(corpus))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "26"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(Data.second.unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "127627 127626\n"
     ]
    }
   ],
   "source": [
    "token_total = [n for n in token_total if len(n)>0]\n",
    "lemm_total = [n for n in lemm_total if len(n)>0]\n",
    "print(len(lemm_total), len(token_total))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1000,) (1000,)\n"
     ]
    }
   ],
   "source": [
    "DataSampled = Data.sample(1000)\n",
    "DataDes=DataSampled.Description.tolist()\n",
    "DataDes = [str(n) for n in DataDes]\n",
    "\n",
    "\n",
    "# #use extend so it's a big flat list of vocab\n",
    "lemmatized = []\n",
    "tokenized = []\n",
    "for i in DataDes:\n",
    "    allwords_stemmed = tokenize_and_lemmatize(i, stopa) #for each item in 'synopses', tokenize/stem\n",
    "    lemmatized.extend([allwords_stemmed]) #extend the 'totalvocab_stemmed' list\n",
    "\n",
    "    allwords_tokenized = tokenize_only(i, stopa)\n",
    "    tokenized.extend([allwords_tokenized])\n",
    "\n",
    "print(np.shape(tokenized),  np.shape(lemmatized))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TF-IDF LSI and LDA models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 372 ms\n"
     ]
    }
   ],
   "source": [
    "%time tfidf = gensim.models.TfidfModel(corpus, normalize = True)\n",
    "corpus_tfidf = tfidf[corpus]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def vectorSimilarity(Model, dictio, MatrixSim, text):\n",
    "    vec_bow = dictio.doc2bow(text)\n",
    "    vec_tfidf = Model[vec_bow]\n",
    "    vectSim = MatrixSim[vec_tfidf]\n",
    "    if str(type(vectSim)) == \"<class 'numpy.ndarray'>\":\n",
    "        return vectSim\n",
    "    else:\n",
    "        return np.asarray(vectSim)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "vectors = []\n",
    "for n, i in enumerate(tokenized):\n",
    "    vec_bow = dictionary.doc2bow(i)\n",
    "    vec_tfidf = tfidf[vec_bow]\n",
    "    vector = gensim.matutils.sparse2full(vec_tfidf, length = len(corpus))\n",
    "    vectors.append(vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.695522383088 0.711751123329 0.73052580772\n"
     ]
    }
   ],
   "source": [
    "DataSampled['vecs'] = vectors\n",
    "X = np.vstack(DataSampled.vecs)\n",
    "kmeans = MiniBatchKMeans(n_clusters = len(Data.second.unique()), batch_size = 100).fit(X)\n",
    "pred_classes = kmeans.predict(X)\n",
    "\n",
    "print(metrics.adjusted_rand_score(DataSampled.second.tolist(), pred_classes.tolist()),\n",
    "      metrics.adjusted_mutual_info_score(DataSampled.second.tolist(), pred_classes.tolist()), \n",
    "      metrics.completeness_score(DataSampled.second.tolist(), pred_classes.tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:gensim.models.lsimodel:using serial LSI version on this node\n",
      "INFO:gensim.models.lsimodel:updating model with new documents\n",
      "INFO:gensim.models.lsimodel:preparing a new chunk of documents\n",
      "INFO:gensim.models.lsimodel:using 100 extra samples and 2 power iterations\n",
      "INFO:gensim.models.lsimodel:1st phase: constructing (1148, 400) action matrix\n",
      "INFO:gensim.models.lsimodel:orthonormalizing (1148, 400) action matrix\n",
      "INFO:gensim.models.lsimodel:2nd phase: running dense svd on (400, 20000) matrix\n",
      "INFO:gensim.models.lsimodel:computing the final decomposition\n",
      "INFO:gensim.models.lsimodel:keeping 176 factors (discarding 0.000% of energy spectrum)\n",
      "INFO:gensim.models.lsimodel:processed documents up to #20000\n",
      "INFO:gensim.models.lsimodel:topic #0(87.702): 0.942*\"clay\" + 0.230*\"gravel\" + 0.227*\"sandy\" + 0.084*\"sand\" + 0.031*\"shale\" + 0.015*\"soil\" + 0.015*\"sandstone\" + 0.011*\"puggy\" + 0.007*\"stoney\" + 0.007*\"solid\"\n",
      "INFO:gensim.models.lsimodel:topic #1(57.330): 0.916*\"shale\" + 0.245*\"sandy\" + -0.245*\"gravel\" + -0.169*\"sand\" + 0.109*\"sandstone\" + 0.017*\"coal\" + -0.016*\"clay\" + 0.016*\"puggy\" + -0.009*\"drift\" + 0.006*\"loam\"\n",
      "INFO:gensim.models.lsimodel:topic #2(55.739): -0.710*\"gravel\" + -0.586*\"sand\" + -0.307*\"shale\" + 0.219*\"clay\" + 0.073*\"sandy\" + -0.054*\"sandstone\" + -0.032*\"drift\" + -0.017*\"river\" + -0.010*\"basalt\" + -0.006*\"coal\"\n",
      "INFO:gensim.models.lsimodel:topic #3(49.668): 0.992*\"sandstone\" + -0.111*\"shale\" + -0.057*\"sandy\" + -0.019*\"gravel\" + 0.016*\"clayey\" + -0.014*\"sand\" + 0.011*\"geologist\" + 0.008*\"conglomerate\" + 0.008*\"shell\" + 0.008*\"clay\"\n",
      "INFO:gensim.models.lsimodel:topic #4(44.465): -0.785*\"sand\" + 0.604*\"gravel\" + -0.104*\"sandy\" + -0.054*\"clay\" + -0.046*\"drift\" + 0.043*\"shale\" + 0.018*\"basalt\" + 0.014*\"river\" + -0.007*\"soil\" + -0.005*\"loam\"\n",
      "INFO:gensim.models.lsimodel:preparing a new chunk of documents\n",
      "INFO:gensim.models.lsimodel:using 100 extra samples and 2 power iterations\n",
      "INFO:gensim.models.lsimodel:1st phase: constructing (1148, 400) action matrix\n",
      "INFO:gensim.models.lsimodel:orthonormalizing (1148, 400) action matrix\n",
      "INFO:gensim.models.lsimodel:2nd phase: running dense svd on (400, 20000) matrix\n",
      "INFO:gensim.models.lsimodel:computing the final decomposition\n",
      "INFO:gensim.models.lsimodel:keeping 300 factors (discarding 0.230% of energy spectrum)\n",
      "INFO:gensim.models.lsimodel:merging projections: (1148, 176) + (1148, 300)\n",
      "INFO:gensim.models.lsimodel:keeping 300 factors (discarding 0.155% of energy spectrum)\n",
      "INFO:gensim.models.lsimodel:processed documents up to #40000\n",
      "INFO:gensim.models.lsimodel:topic #0(150.615): -0.838*\"clay\" + -0.414*\"gravel\" + -0.272*\"sandy\" + -0.214*\"sand\" + -0.039*\"silty\" + -0.034*\"rounded\" + -0.023*\"jasper\" + -0.023*\"quartz\" + -0.019*\"silica\" + -0.015*\"angular\"\n",
      "INFO:gensim.models.lsimodel:topic #1(108.629): 0.684*\"gravel\" + 0.550*\"sand\" + -0.432*\"clay\" + -0.170*\"sandy\" + 0.051*\"rounded\" + 0.046*\"quartz\" + 0.041*\"jasper\" + 0.038*\"clayey\" + 0.035*\"silty\" + 0.025*\"angular\"\n",
      "INFO:gensim.models.lsimodel:topic #2(74.912): 0.794*\"sand\" + -0.592*\"gravel\" + 0.071*\"sandy\" + 0.059*\"clay\" + 0.050*\"silty\" + 0.039*\"clayey\" + 0.036*\"silica\" + 0.031*\"shale\" + 0.026*\"rounded\" + 0.023*\"graded\"\n",
      "INFO:gensim.models.lsimodel:topic #3(67.226): 0.698*\"shale\" + 0.666*\"sandy\" + -0.244*\"clay\" + 0.068*\"sandstone\" + 0.048*\"gravel\" + -0.036*\"sand\" + 0.020*\"loam\" + 0.018*\"rounded\" + 0.015*\"jasper\" + 0.013*\"quartz\"\n",
      "INFO:gensim.models.lsimodel:topic #4(60.316): -0.687*\"shale\" + 0.660*\"sandy\" + -0.212*\"clay\" + -0.208*\"sandstone\" + 0.026*\"loam\" + 0.025*\"rounded\" + 0.020*\"jasper\" + 0.019*\"soil\" + -0.016*\"puggy\" + -0.016*\"coal\"\n",
      "INFO:gensim.models.lsimodel:preparing a new chunk of documents\n",
      "INFO:gensim.models.lsimodel:using 100 extra samples and 2 power iterations\n",
      "INFO:gensim.models.lsimodel:1st phase: constructing (1148, 400) action matrix\n",
      "INFO:gensim.models.lsimodel:orthonormalizing (1148, 400) action matrix\n",
      "INFO:gensim.models.lsimodel:2nd phase: running dense svd on (400, 20000) matrix\n",
      "INFO:gensim.models.lsimodel:computing the final decomposition\n",
      "INFO:gensim.models.lsimodel:keeping 300 factors (discarding 0.382% of energy spectrum)\n",
      "INFO:gensim.models.lsimodel:merging projections: (1148, 300) + (1148, 300)\n",
      "INFO:gensim.models.lsimodel:keeping 300 factors (discarding 0.507% of energy spectrum)\n",
      "INFO:gensim.models.lsimodel:processed documents up to #60000\n",
      "INFO:gensim.models.lsimodel:topic #0(211.843): 0.707*\"clay\" + 0.522*\"gravel\" + 0.298*\"sand\" + 0.228*\"sandy\" + 0.150*\"quartz\" + 0.115*\"jasper\" + 0.107*\"angular\" + 0.098*\"silty\" + 0.073*\"subrounded\" + 0.063*\"chert\"\n",
      "INFO:gensim.models.lsimodel:topic #1(149.472): -0.622*\"clay\" + 0.497*\"gravel\" + 0.437*\"sand\" + -0.224*\"sandy\" + 0.200*\"quartz\" + 0.144*\"jasper\" + 0.132*\"angular\" + 0.091*\"subrounded\" + 0.085*\"clear\" + 0.081*\"chert\"\n",
      "INFO:gensim.models.lsimodel:topic #2(108.998): 0.493*\"quartz\" + 0.333*\"angular\" + 0.325*\"jasper\" + -0.317*\"gravel\" + -0.300*\"sand\" + 0.250*\"clear\" + 0.197*\"sandy\" + 0.196*\"subangular\" + 0.185*\"subrounded\" + 0.177*\"chert\"\n",
      "INFO:gensim.models.lsimodel:topic #3(95.862): -0.780*\"sand\" + 0.609*\"gravel\" + -0.093*\"clay\" + -0.073*\"silty\" + -0.038*\"clayey\" + -0.035*\"quartz\" + 0.031*\"claybound\" + -0.024*\"clear\" + -0.021*\"angular\" + -0.020*\"silt\"\n",
      "INFO:gensim.models.lsimodel:topic #4(81.968): -0.895*\"sandy\" + 0.308*\"clay\" + -0.238*\"shale\" + 0.098*\"quartz\" + -0.097*\"sand\" + 0.071*\"jasper\" + 0.060*\"subrounded\" + -0.060*\"gravel\" + 0.053*\"chert\" + 0.050*\"angular\"\n",
      "INFO:gensim.models.lsimodel:preparing a new chunk of documents\n",
      "INFO:gensim.models.lsimodel:using 100 extra samples and 2 power iterations\n",
      "INFO:gensim.models.lsimodel:1st phase: constructing (1148, 400) action matrix\n",
      "INFO:gensim.models.lsimodel:orthonormalizing (1148, 400) action matrix\n",
      "INFO:gensim.models.lsimodel:2nd phase: running dense svd on (400, 20000) matrix\n",
      "INFO:gensim.models.lsimodel:computing the final decomposition\n",
      "INFO:gensim.models.lsimodel:keeping 295 factors (discarding 0.000% of energy spectrum)\n",
      "INFO:gensim.models.lsimodel:merging projections: (1148, 300) + (1148, 295)\n",
      "INFO:gensim.models.lsimodel:keeping 300 factors (discarding 0.161% of energy spectrum)\n",
      "INFO:gensim.models.lsimodel:processed documents up to #80000\n",
      "INFO:gensim.models.lsimodel:topic #0(235.033): 0.759*\"clay\" + 0.494*\"gravel\" + 0.281*\"sand\" + 0.224*\"sandy\" + 0.111*\"quartz\" + 0.086*\"jasper\" + 0.085*\"silty\" + 0.079*\"angular\" + 0.055*\"subrounded\" + 0.046*\"chert\"\n",
      "INFO:gensim.models.lsimodel:topic #1(165.942): 0.570*\"gravel\" + -0.567*\"clay\" + 0.479*\"sand\" + -0.207*\"sandy\" + 0.159*\"quartz\" + 0.116*\"jasper\" + 0.105*\"angular\" + 0.074*\"subrounded\" + 0.065*\"clear\" + 0.065*\"chert\"\n",
      "INFO:gensim.models.lsimodel:topic #2(114.096): 0.503*\"quartz\" + 0.341*\"angular\" + 0.338*\"jasper\" + -0.299*\"sand\" + 0.248*\"clear\" + 0.237*\"sandy\" + 0.197*\"subangular\" + 0.194*\"subrounded\" + 0.193*\"basalt\" + 0.185*\"chert\"\n",
      "INFO:gensim.models.lsimodel:topic #3(107.487): 0.763*\"sand\" + -0.622*\"gravel\" + 0.076*\"quartz\" + 0.075*\"clay\" + 0.075*\"silty\" + 0.048*\"angular\" + 0.043*\"clayey\" + 0.042*\"clear\" + 0.035*\"jasper\" + 0.031*\"subangular\"\n",
      "INFO:gensim.models.lsimodel:topic #4(91.288): -0.882*\"sandy\" + 0.282*\"clay\" + -0.271*\"shale\" + 0.124*\"quartz\" + -0.096*\"sand\" + 0.087*\"jasper\" + 0.073*\"angular\" + -0.070*\"gravel\" + 0.064*\"subrounded\" + 0.060*\"silty\"\n",
      "INFO:gensim.models.lsimodel:preparing a new chunk of documents\n",
      "INFO:gensim.models.lsimodel:using 100 extra samples and 2 power iterations\n",
      "INFO:gensim.models.lsimodel:1st phase: constructing (1148, 400) action matrix\n",
      "INFO:gensim.models.lsimodel:orthonormalizing (1148, 400) action matrix\n",
      "INFO:gensim.models.lsimodel:2nd phase: running dense svd on (400, 20000) matrix\n",
      "INFO:gensim.models.lsimodel:computing the final decomposition\n",
      "INFO:gensim.models.lsimodel:keeping 292 factors (discarding 0.000% of energy spectrum)\n",
      "INFO:gensim.models.lsimodel:merging projections: (1148, 300) + (1148, 292)\n",
      "INFO:gensim.models.lsimodel:keeping 300 factors (discarding 0.232% of energy spectrum)\n",
      "INFO:gensim.models.lsimodel:processed documents up to #100000\n",
      "INFO:gensim.models.lsimodel:topic #0(251.042): 0.783*\"clay\" + 0.471*\"gravel\" + 0.278*\"sand\" + 0.226*\"sandy\" + 0.092*\"quartz\" + 0.081*\"silty\" + 0.071*\"jasper\" + 0.066*\"angular\" + 0.046*\"subrounded\" + 0.039*\"chert\"\n",
      "INFO:gensim.models.lsimodel:topic #1(180.861): 0.595*\"gravel\" + -0.538*\"clay\" + 0.516*\"sand\" + -0.193*\"sandy\" + 0.127*\"quartz\" + 0.093*\"jasper\" + 0.084*\"angular\" + 0.060*\"subrounded\" + 0.052*\"chert\" + 0.051*\"clear\"\n",
      "INFO:gensim.models.lsimodel:topic #2(118.810): -0.708*\"sand\" + 0.339*\"gravel\" + 0.320*\"quartz\" + 0.226*\"jasper\" + 0.219*\"angular\" + 0.161*\"sandy\" + 0.155*\"basalt\" + 0.152*\"clear\" + 0.129*\"subrounded\" + -0.128*\"clay\"\n",
      "INFO:gensim.models.lsimodel:topic #3(115.735): -0.547*\"gravel\" + 0.397*\"quartz\" + 0.373*\"sand\" + 0.267*\"angular\" + 0.257*\"jasper\" + 0.214*\"sandy\" + 0.196*\"clear\" + 0.174*\"silty\" + 0.163*\"basalt\" + 0.155*\"subangular\"\n",
      "INFO:gensim.models.lsimodel:topic #4(98.871): -0.829*\"sandy\" + -0.404*\"shale\" + 0.255*\"clay\" + 0.141*\"quartz\" + 0.098*\"jasper\" + 0.087*\"angular\" + 0.081*\"silty\" + 0.079*\"basalt\" + -0.077*\"gravel\" + -0.071*\"sand\"\n",
      "INFO:gensim.models.lsimodel:preparing a new chunk of documents\n",
      "INFO:gensim.models.lsimodel:using 100 extra samples and 2 power iterations\n",
      "INFO:gensim.models.lsimodel:1st phase: constructing (1148, 400) action matrix\n",
      "INFO:gensim.models.lsimodel:orthonormalizing (1148, 400) action matrix\n",
      "INFO:gensim.models.lsimodel:2nd phase: running dense svd on (400, 20000) matrix\n",
      "INFO:gensim.models.lsimodel:computing the final decomposition\n",
      "INFO:gensim.models.lsimodel:keeping 300 factors (discarding 0.097% of energy spectrum)\n",
      "INFO:gensim.models.lsimodel:merging projections: (1148, 300) + (1148, 300)\n",
      "INFO:gensim.models.lsimodel:keeping 300 factors (discarding 0.265% of energy spectrum)\n",
      "INFO:gensim.models.lsimodel:processed documents up to #120000\n",
      "INFO:gensim.models.lsimodel:topic #0(264.436): 0.804*\"clay\" + 0.450*\"gravel\" + 0.272*\"sand\" + 0.220*\"sandy\" + 0.079*\"quartz\" + 0.078*\"silty\" + 0.061*\"jasper\" + 0.057*\"angular\" + 0.040*\"subrounded\" + 0.033*\"chert\"\n",
      "INFO:gensim.models.lsimodel:topic #1(193.407): 0.616*\"gravel\" + 0.532*\"sand\" + -0.517*\"clay\" + -0.169*\"sandy\" + 0.109*\"quartz\" + 0.081*\"jasper\" + 0.073*\"angular\" + 0.052*\"subrounded\" + 0.045*\"chert\" + 0.044*\"clayey\"\n",
      "INFO:gensim.models.lsimodel:topic #2(125.982): -0.789*\"sand\" + 0.544*\"gravel\" + 0.130*\"quartz\" + -0.100*\"clay\" + 0.100*\"jasper\" + 0.091*\"basalt\" + 0.091*\"angular\" + 0.072*\"sandy\" + 0.058*\"clear\" + 0.058*\"chert\"\n",
      "INFO:gensim.models.lsimodel:topic #3(118.953): 0.478*\"quartz\" + -0.334*\"gravel\" + 0.323*\"angular\" + 0.318*\"jasper\" + 0.295*\"sandy\" + 0.280*\"basalt\" + 0.231*\"clear\" + 0.188*\"silty\" + 0.187*\"subangular\" + 0.185*\"subrounded\"\n",
      "INFO:gensim.models.lsimodel:topic #4(105.077): -0.674*\"shale\" + -0.653*\"sandy\" + 0.187*\"clay\" + 0.138*\"quartz\" + 0.119*\"basalt\" + 0.096*\"jasper\" + 0.088*\"angular\" + 0.075*\"silty\" + -0.065*\"gravel\" + 0.063*\"subrounded\"\n",
      "INFO:gensim.models.lsimodel:preparing a new chunk of documents\n",
      "INFO:gensim.models.lsimodel:using 100 extra samples and 2 power iterations\n",
      "INFO:gensim.models.lsimodel:1st phase: constructing (1148, 400) action matrix\n",
      "INFO:gensim.models.lsimodel:orthonormalizing (1148, 400) action matrix\n",
      "INFO:gensim.models.lsimodel:2nd phase: running dense svd on (400, 7626) matrix\n",
      "INFO:gensim.models.lsimodel:computing the final decomposition\n",
      "INFO:gensim.models.lsimodel:keeping 300 factors (discarding 0.402% of energy spectrum)\n",
      "INFO:gensim.models.lsimodel:merging projections: (1148, 300) + (1148, 300)\n",
      "INFO:gensim.models.lsimodel:keeping 300 factors (discarding 0.332% of energy spectrum)\n",
      "INFO:gensim.models.lsimodel:processed documents up to #127626\n",
      "INFO:gensim.models.lsimodel:topic #0(270.382): 0.804*\"clay\" + 0.448*\"gravel\" + 0.278*\"sand\" + 0.217*\"sandy\" + 0.082*\"silty\" + 0.075*\"quartz\" + 0.058*\"jasper\" + 0.055*\"angular\" + 0.038*\"subrounded\" + 0.032*\"chert\"\n",
      "INFO:gensim.models.lsimodel:topic #1(197.671): 0.615*\"gravel\" + 0.536*\"sand\" + -0.520*\"clay\" + -0.166*\"sandy\" + 0.102*\"quartz\" + 0.076*\"jasper\" + 0.069*\"angular\" + 0.050*\"subrounded\" + 0.046*\"clayey\" + 0.042*\"chert\"\n",
      "INFO:gensim.models.lsimodel:topic #2(129.284): -0.786*\"sand\" + 0.566*\"gravel\" + 0.107*\"quartz\" + -0.095*\"clay\" + 0.084*\"jasper\" + 0.083*\"basalt\" + 0.076*\"angular\" + 0.064*\"sandy\" + 0.049*\"chert\" + 0.047*\"subrounded\"\n",
      "INFO:gensim.models.lsimodel:topic #3(120.160): 0.477*\"quartz\" + 0.320*\"angular\" + 0.315*\"jasper\" + 0.313*\"basalt\" + 0.312*\"sandy\" + -0.300*\"gravel\" + 0.228*\"clear\" + 0.188*\"silty\" + 0.185*\"subangular\" + 0.184*\"subrounded\"\n",
      "INFO:gensim.models.lsimodel:topic #4(107.481): -0.710*\"shale\" + -0.614*\"sandy\" + 0.173*\"clay\" + 0.140*\"basalt\" + 0.139*\"quartz\" + 0.095*\"jasper\" + 0.089*\"angular\" + 0.077*\"silty\" + 0.063*\"clear\" + 0.063*\"subrounded\"\n"
     ]
    }
   ],
   "source": [
    "lsi = gensim.models.LsiModel(corpus, id2word=dictionary, num_topics=300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "vectors = []\n",
    "for n, i in enumerate(tokenized):\n",
    "    vec_bow = dictionary.doc2bow(i)\n",
    "    vec_lsi = lsi[vec_bow]\n",
    "    vector = gensim.matutils.sparse2full(vec_lsi, length = 300)\n",
    "    vectors.append(vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bokeh.core.state:Session output file 'plot.html' already exists, will be overwritten.\n"
     ]
    }
   ],
   "source": [
    "plot_embedding([x for x in vectors[:1000]],\n",
    "               [';'.join(x) for x in tokenized[:1000]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:gensim.models.ldamodel:using symmetric alpha at 0.02\n",
      "INFO:gensim.models.ldamodel:using symmetric eta at 0.0008710801393728223\n",
      "INFO:gensim.models.ldamodel:using serial LDA version on this node\n",
      "INFO:gensim.models.ldamulticore:running online LDA training, 50 topics, 2 passes over the supplied corpus of 127626 documents, updating every 80000 documents, evaluating every ~127626 documents, iterating 50x with a convergence threshold of 0.001000\n",
      "WARNING:gensim.models.ldamulticore:too few updates, training might not converge; consider increasing the number of passes or iterations to improve accuracy\n",
      "INFO:gensim.models.ldamulticore:training LDA model using 8 processes\n",
      "INFO:gensim.models.ldamulticore:PROGRESS: pass 0, dispatched chunk #0 = documents up to #10000/127626, outstanding queue size 1\n",
      "INFO:gensim.models.ldamulticore:PROGRESS: pass 0, dispatched chunk #1 = documents up to #20000/127626, outstanding queue size 2\n",
      "INFO:gensim.models.ldamulticore:PROGRESS: pass 0, dispatched chunk #2 = documents up to #30000/127626, outstanding queue size 3\n",
      "INFO:gensim.models.ldamulticore:PROGRESS: pass 0, dispatched chunk #3 = documents up to #40000/127626, outstanding queue size 4\n",
      "INFO:gensim.models.ldamulticore:PROGRESS: pass 0, dispatched chunk #4 = documents up to #50000/127626, outstanding queue size 5\n",
      "INFO:gensim.models.ldamulticore:PROGRESS: pass 0, dispatched chunk #5 = documents up to #60000/127626, outstanding queue size 6\n",
      "INFO:gensim.models.ldamulticore:PROGRESS: pass 0, dispatched chunk #6 = documents up to #70000/127626, outstanding queue size 7\n",
      "INFO:gensim.models.ldamulticore:PROGRESS: pass 0, dispatched chunk #7 = documents up to #80000/127626, outstanding queue size 8\n",
      "INFO:gensim.models.ldamulticore:PROGRESS: pass 0, dispatched chunk #8 = documents up to #90000/127626, outstanding queue size 9\n",
      "INFO:gensim.models.ldamulticore:PROGRESS: pass 0, dispatched chunk #9 = documents up to #100000/127626, outstanding queue size 10\n",
      "INFO:gensim.models.ldamulticore:PROGRESS: pass 0, dispatched chunk #10 = documents up to #110000/127626, outstanding queue size 11\n",
      "INFO:gensim.models.ldamulticore:PROGRESS: pass 0, dispatched chunk #11 = documents up to #120000/127626, outstanding queue size 12\n",
      "INFO:gensim.models.ldamulticore:PROGRESS: pass 0, dispatched chunk #12 = documents up to #127626/127626, outstanding queue size 13\n",
      "INFO:gensim.models.ldamodel:merging changes from 80000 documents into a model of 127626 documents\n",
      "INFO:gensim.models.ldamodel:topic #46 (0.020): 0.155*\"gravel\" + 0.112*\"clayey\" + 0.070*\"clay\" + 0.050*\"sand\" + 0.047*\"claybound\" + 0.041*\"sandy\" + 0.040*\"overall\" + 0.040*\"soil\" + 0.033*\"topsoil\" + 0.031*\"quartz\"\n",
      "INFO:gensim.models.ldamodel:topic #47 (0.020): 0.191*\"shale\" + 0.114*\"clay\" + 0.102*\"sandy\" + 0.052*\"slate\" + 0.044*\"gravel\" + 0.043*\"puggy\" + 0.034*\"sandstone\" + 0.033*\"weathered\" + 0.020*\"mixed\" + 0.019*\"basalt\"\n",
      "INFO:gensim.models.ldamodel:topic #30 (0.020): 0.257*\"basalt\" + 0.094*\"sand\" + 0.071*\"granite\" + 0.043*\"sandy\" + 0.029*\"clay\" + 0.021*\"agglomerate\" + 0.020*\"pale\" + 0.016*\"angular\" + 0.015*\"mixed\" + 0.014*\"gravel\"\n",
      "INFO:gensim.models.ldamodel:topic #18 (0.020): 0.366*\"sandy\" + 0.145*\"clay\" + 0.098*\"gravel\" + 0.066*\"sand\" + 0.040*\"slightly\" + 0.029*\"shale\" + 0.011*\"jasper\" + 0.010*\"quartz\" + 0.010*\"soil\" + 0.010*\"mainly\"\n",
      "INFO:gensim.models.ldamodel:topic #10 (0.020): 0.371*\"soil\" + 0.144*\"clay\" + 0.122*\"sandy\" + 0.022*\"gravel\" + 0.021*\"mudstone\" + 0.019*\"quartz\" + 0.017*\"angular\" + 0.015*\"silty\" + 0.012*\"overall\" + 0.011*\"jasper\"\n",
      "INFO:gensim.models.ldamodel:topic diff=42.429780, rho=1.000000\n",
      "INFO:gensim.models.ldamodel:merging changes from 47626 documents into a model of 127626 documents\n",
      "INFO:gensim.models.ldamodel:topic #3 (0.020): 0.205*\"sandy\" + 0.170*\"gravel\" + 0.099*\"clay\" + 0.070*\"sand\" + 0.049*\"sandstone\" + 0.045*\"topsoil\" + 0.040*\"siltstone\" + 0.027*\"minor\" + 0.016*\"trace\" + 0.016*\"silty\"\n",
      "INFO:gensim.models.ldamodel:topic #43 (0.020): 0.151*\"sand\" + 0.127*\"clayey\" + 0.101*\"silty\" + 0.078*\"clay\" + 0.063*\"sandy\" + 0.052*\"claystone\" + 0.034*\"gravel\" + 0.020*\"slightly\" + 0.014*\"pebbly\" + 0.013*\"quartz\"\n",
      "INFO:gensim.models.ldamodel:topic #19 (0.020): 0.360*\"sand\" + 0.321*\"gravel\" + 0.064*\"shale\" + 0.052*\"clay\" + 0.038*\"granite\" + 0.019*\"loam\" + 0.019*\"basalt\" + 0.015*\"clayey\" + 0.010*\"drift\" + 0.007*\"silt\"\n",
      "INFO:gensim.models.ldamodel:topic #29 (0.020): 0.194*\"clay\" + 0.105*\"granite\" + 0.068*\"sand\" + 0.063*\"loam\" + 0.041*\"sandy\" + 0.026*\"basaltic\" + 0.025*\"soil\" + 0.023*\"slightly\" + 0.022*\"shaley\" + 0.021*\"coal\"\n",
      "INFO:gensim.models.ldamodel:topic #34 (0.020): 0.274*\"basalt\" + 0.123*\"gravel\" + 0.073*\"silty\" + 0.030*\"conglomerate\" + 0.027*\"dirty\" + 0.027*\"shale\" + 0.026*\"clay\" + 0.022*\"loam\" + 0.021*\"topsoil\" + 0.020*\"granite\"\n",
      "INFO:gensim.models.ldamodel:topic diff=2.560204, rho=0.333333\n",
      "INFO:gensim.models.ldamodel:-6.624 per-word bound, 98.7 perplexity estimate based on a held-out corpus of 7626 documents with 10192 words\n",
      "INFO:gensim.models.ldamulticore:PROGRESS: pass 1, dispatched chunk #0 = documents up to #10000/127626, outstanding queue size 1\n",
      "INFO:gensim.models.ldamulticore:PROGRESS: pass 1, dispatched chunk #1 = documents up to #20000/127626, outstanding queue size 2\n",
      "INFO:gensim.models.ldamulticore:PROGRESS: pass 1, dispatched chunk #2 = documents up to #30000/127626, outstanding queue size 3\n",
      "INFO:gensim.models.ldamulticore:PROGRESS: pass 1, dispatched chunk #3 = documents up to #40000/127626, outstanding queue size 4\n",
      "INFO:gensim.models.ldamulticore:PROGRESS: pass 1, dispatched chunk #4 = documents up to #50000/127626, outstanding queue size 5\n",
      "INFO:gensim.models.ldamulticore:PROGRESS: pass 1, dispatched chunk #5 = documents up to #60000/127626, outstanding queue size 6\n",
      "INFO:gensim.models.ldamulticore:PROGRESS: pass 1, dispatched chunk #6 = documents up to #70000/127626, outstanding queue size 7\n",
      "INFO:gensim.models.ldamulticore:PROGRESS: pass 1, dispatched chunk #7 = documents up to #80000/127626, outstanding queue size 8\n",
      "INFO:gensim.models.ldamulticore:PROGRESS: pass 1, dispatched chunk #8 = documents up to #90000/127626, outstanding queue size 9\n",
      "INFO:gensim.models.ldamulticore:PROGRESS: pass 1, dispatched chunk #9 = documents up to #100000/127626, outstanding queue size 10\n",
      "INFO:gensim.models.ldamulticore:PROGRESS: pass 1, dispatched chunk #10 = documents up to #110000/127626, outstanding queue size 11\n",
      "INFO:gensim.models.ldamulticore:PROGRESS: pass 1, dispatched chunk #11 = documents up to #120000/127626, outstanding queue size 12\n",
      "INFO:gensim.models.ldamulticore:PROGRESS: pass 1, dispatched chunk #12 = documents up to #127626/127626, outstanding queue size 13\n",
      "INFO:gensim.models.ldamodel:merging changes from 80000 documents into a model of 127626 documents\n",
      "INFO:gensim.models.ldamodel:topic #37 (0.020): 0.280*\"shale\" + 0.106*\"sandstone\" + 0.104*\"gravel\" + 0.097*\"clay\" + 0.058*\"gravelly\" + 0.034*\"claybound\" + 0.028*\"sand\" + 0.022*\"sandy\" + 0.020*\"topsoil\" + 0.015*\"silty\"\n",
      "INFO:gensim.models.ldamodel:topic #33 (0.020): 0.385*\"clay\" + 0.295*\"gravel\" + 0.095*\"sandy\" + 0.069*\"sand\" + 0.053*\"basalt\" + 0.018*\"sandstone\" + 0.011*\"silty\" + 0.007*\"granite\" + 0.003*\"shale\" + 0.003*\"topsoil\"\n",
      "INFO:gensim.models.ldamodel:topic #5 (0.020): 0.513*\"sandstone\" + 0.191*\"clay\" + 0.045*\"granite\" + 0.034*\"stoney\" + 0.030*\"sandy\" + 0.024*\"gravel\" + 0.023*\"shale\" + 0.022*\"sand\" + 0.012*\"topsoil\" + 0.008*\"dirty\"\n",
      "INFO:gensim.models.ldamodel:topic #25 (0.020): 0.222*\"shale\" + 0.085*\"gravel\" + 0.065*\"sand\" + 0.055*\"clayey\" + 0.042*\"granite\" + 0.036*\"basalt\" + 0.028*\"weathered\" + 0.026*\"sandy\" + 0.021*\"quartz\" + 0.020*\"minor\"\n",
      "INFO:gensim.models.ldamodel:topic #15 (0.020): 0.137*\"gravel\" + 0.097*\"overall\" + 0.067*\"none\" + 0.062*\"clay\" + 0.060*\"sand\" + 0.043*\"subrounded\" + 0.032*\"gravelly\" + 0.029*\"clayey\" + 0.028*\"silt\" + 0.028*\"cream\"\n",
      "INFO:gensim.models.ldamodel:topic diff=0.286696, rho=0.260267\n",
      "INFO:gensim.models.ldamodel:merging changes from 47626 documents into a model of 127626 documents\n",
      "INFO:gensim.models.ldamodel:topic #17 (0.020): 0.350*\"shale\" + 0.160*\"coal\" + 0.099*\"sandy\" + 0.093*\"sand\" + 0.027*\"clay\" + 0.025*\"granite\" + 0.022*\"brwon\" + 0.013*\"washed\" + 0.012*\"gravel\" + 0.010*\"weathered\"\n",
      "INFO:gensim.models.ldamodel:topic #32 (0.020): 0.095*\"clay\" + 0.043*\"gravel\" + 0.039*\"overall\" + 0.038*\"silty\" + 0.035*\"jasper\" + 0.034*\"solid\" + 0.034*\"quartz\" + 0.034*\"subangular\" + 0.033*\"sand\" + 0.032*\"pale\"\n",
      "INFO:gensim.models.ldamodel:topic #25 (0.020): 0.198*\"shale\" + 0.076*\"gravel\" + 0.060*\"sand\" + 0.051*\"clayey\" + 0.045*\"granite\" + 0.033*\"weathered\" + 0.032*\"basalt\" + 0.023*\"sandy\" + 0.023*\"minor\" + 0.019*\"quartz\"\n",
      "INFO:gensim.models.ldamodel:topic #41 (0.020): 0.321*\"clay\" + 0.243*\"gravel\" + 0.095*\"sandy\" + 0.069*\"sand\" + 0.057*\"coal\" + 0.035*\"soil\" + 0.009*\"shale\" + 0.006*\"except\" + 0.006*\"conglomerate\" + 0.005*\"jasper\"\n",
      "INFO:gensim.models.ldamodel:topic #39 (0.020): 0.083*\"quartz\" + 0.053*\"angular\" + 0.050*\"sand\" + 0.049*\"jasper\" + 0.042*\"soil\" + 0.037*\"gravel\" + 0.034*\"clear\" + 0.033*\"chert\" + 0.031*\"graded\" + 0.029*\"subrounded\"\n",
      "INFO:gensim.models.ldamodel:topic diff=0.259912, rho=0.260267\n",
      "INFO:gensim.models.ldamodel:-6.109 per-word bound, 69.0 perplexity estimate based on a held-out corpus of 7626 documents with 10192 words\n"
     ]
    }
   ],
   "source": [
    "lda = gensim.models.LdaMulticore(corpus_tfidf, id2word=dictionary, num_topics=50, chunksize=10000, passes=2, workers = 8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "vectors = []\n",
    "for n, i in enumerate(tokenized):\n",
    "    vec_bow = dictionary.doc2bow(i)\n",
    "    vec_lda = lda[vec_bow]\n",
    "    vector = gensim.matutils.sparse2full(vec_lda, length = 50)\n",
    "    vectors.append(vector)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dictionary from papers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "tfidfSim = gensim.similarities.Similarity(root+'/tmp/',corpus_tfidf, len(dictionary))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "vector =vectorSimilarity(tfidf, dictionary, tfidfSim, lemm_total[0])\n",
    "n_rows = vector.shape[0]\n",
    "chunck_size = (1000, 1000)\n",
    "\n",
    "\n",
    "with h5py.File('tmp/tfidf.h5', 'w') as f:\n",
    "    dset= f.create_dataset('chunked', shape = (n_rows, n_rows), chunks = chunck_size)\n",
    "    for n, i in enumerate(lemm_total):\n",
    "        dset[n, :] = vectorSimilarity(tfidf, dictionary, tfidfSim, i)\n",
    "\n",
    "#     last_row = dset[-1, :].copy()\n",
    "    \n",
    "# # Read file\n",
    "# test = h5py.File('tmp/test.h5', 'r')\n",
    "# data = test['chunked']\n",
    "# np.all(data[-1, :] == last_row)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Read file\n",
    "test = h5py.File('tmp/tfidf.h5', 'r')\n",
    "data = test['chunked']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import dask.array as da\n",
    "darray = da.from_array(data, chunks=(1000, 1000))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "mds = MDS(n_components=2, dissimilarity=\"precomputed\", random_state=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pos = mds.fit_transform(darray)  # shape (n_components, n_samples)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "n_rows = 100\n",
    "chuck_size = (10, 10)  # Cambia esto a (100, 100) o algo así\n",
    "with h5py.File('/tmp/test.h5', 'w') as f:\n",
    "    dset = f.create_dataset('data', (n_rows, n_rows), chunks=chuck_size)\n",
    "    for i in range(n_rows):\n",
    "        dset[i, :] = similarity()  # Acá es donde tienes que calcular la similarity\n",
    "    last_row = dset[-1, :].copy()\n",
    " \n",
    "# Read file\n",
    "test = h5py.File('/tmp/test.h5', 'r')\n",
    "data = test['data']\n",
    "np.all(data[-1, :] == last_row)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(127930,) float32\n"
     ]
    }
   ],
   "source": [
    "vec_bow = dictionary.doc2bow(lemm_total[0])\n",
    "vec_tfidf = tfidf[vec_bow]\n",
    "sims1 = tfidfSim[vec_tfidf]\n",
    "\n",
    "dtype = sims1.dtype\n",
    "print(sims1.shape, dtype)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['sand']\n",
      "[(1047, 1.0)]\n",
      "['clay']\n",
      "[(217, 1.0)]\n",
      "['drift']\n",
      "[(290, 1.0)]\n",
      "['clay', 'sand']\n",
      "[(217, 0.44255164661160346), (1047, 0.8967430178603893)]\n",
      "['clay']\n",
      "[(217, 1.0)]\n",
      "['clay', 'sandy']\n",
      "[(217, 0.35543402011792474), (1013, 0.9347013733502325)]\n",
      "['drift']\n",
      "[(290, 1.0)]\n",
      "['sand', 'drift']\n",
      "[(290, 0.9636659480506097), (1047, 0.26711035279022716)]\n",
      "['sandstone']\n",
      "[(834, 1.0)]\n",
      "['sandstone']\n",
      "[(834, 1.0)]\n",
      "['sandstone']\n",
      "[(834, 1.0)]\n",
      "['clay', 'sandy']\n",
      "[(217, 0.35543402011792474), (1013, 0.9347013733502325)]\n",
      "['clay']\n",
      "[(217, 1.0)]\n",
      "['clay', 'sandy']\n",
      "[(217, 0.35543402011792474), (1013, 0.9347013733502325)]\n",
      "['sand', 'drift']\n",
      "[(290, 0.9636659480506097), (1047, 0.26711035279022716)]\n",
      "['sand']\n",
      "[(1047, 1.0)]\n",
      "['clay', 'sand']\n",
      "[(217, 0.44255164661160346), (1047, 0.8967430178603893)]\n",
      "['clay', 'boulder']\n",
      "[(217, 0.1698892421345605), (799, 0.9854631628868451)]\n",
      "['sandstone']\n",
      "[(834, 1.0)]\n",
      "['clay']\n",
      "[(217, 1.0)]\n",
      "['clay', 'drift']\n",
      "[(217, 0.13552969245277532), (290, 0.9907732850978855)]\n",
      "['clay', 'pebble', 'pebbly']\n",
      "[(217, 0.10098920304260228), (793, 0.6572207965345351), (914, 0.7469016036074171)]\n",
      "['clay']\n",
      "[(217, 1.0)]\n",
      "['clay']\n",
      "[(217, 1.0)]\n",
      "['clay', 'sand']\n",
      "[(217, 0.44255164661160346), (1047, 0.8967430178603893)]\n",
      "['clay']\n",
      "[(217, 1.0)]\n",
      "['clay']\n",
      "[(217, 1.0)]\n",
      "['sand', 'drift']\n",
      "[(290, 0.9636659480506097), (1047, 0.26711035279022716)]\n",
      "['clay']\n",
      "[(217, 1.0)]\n",
      "['clay', 'streak']\n",
      "[(217, 0.13237258092090137), (282, 0.9912000301757157)]\n"
     ]
    }
   ],
   "source": [
    "# tfidfSimMatrix =[]\n",
    "\n",
    "simsss = []\n",
    "# vectors = []\n",
    "#with h5py.File('tmp/tfidf.hdf5'.format(n), 'w') as f:\n",
    "for n, i in enumerate(lemm_total[:30]):\n",
    "    print(i)\n",
    "#         print(i, n)\n",
    "    vec_bow = dictionary.doc2bow(i)\n",
    "#         print(vec_bow)\n",
    "    vec_tfidf = tfidf[vec_bow]\n",
    "    print(vec_tfidf)\n",
    "#    simsss.append(gensim.matutils.corpus2dense(vec_tfidf, num_terms=300))\n",
    "#         print(vec_tfidf)\n",
    "        \n",
    "#     original = [0]*len(dictionary)\n",
    "#     replaceIndex = [n[0] for n in vec_tfidf]\n",
    "#     replaceValue = [n[1] for n in vec_tfidf]\n",
    "#     for b, c in zip(replaceIndex, replaceValue):\n",
    "#         setitem(original, b, c)\n",
    "#     vectors.append(original)\n",
    "\n",
    "\n",
    "#        sims1 = tfidfSim[vec_tfidf]\n",
    "    \n",
    "    \n",
    "#         print(len(sims1))\n",
    "#         simsss.append(sims1)\n",
    "\n",
    "\n",
    "#        f.create_dataset('tfidf_{}'.format(n), data = sims1)\n",
    "    \n",
    "    \n",
    "#     if (n>0) & (n%10 == 0):\n",
    "#         simsss = np.asarray(simsss)\n",
    "#        \n",
    "#             f.create_dataset('tfidf_{}'.format(n), data = simsss)\n",
    "#             print(f.shape)\n",
    "#             f.close()\n",
    "            \n",
    "#         simsss = []\n",
    "#     elif n == len(lemm_total):\n",
    "#         simsss = np.asarray(simsss)\n",
    "#         with h5py.File('tmp/tfidf_{}.hdf5'.format(n), 'w') as f:\n",
    "#             f.create_dataset('tfidf_{}'.format(n), data = simsss)\n",
    "#             print(f.shape)\n",
    "#             f.close()\n",
    "            \n",
    "#         simsss = []\n",
    "    \n",
    "# simsss.shape\n",
    "#     print(sims1)\n",
    "#     tfidfSimMatrix.append(sims1)\n",
    "# tfidfDisMatrix = 1- np.array(tfidfSimMatrix) \n",
    "\n",
    "\n",
    "#f.close()\n",
    "#print(len(simsss), simsss[0])                "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0 0]\n",
      " [1 4]\n",
      " [6 8]\n",
      " [6 9]\n",
      " [5 8]]\n",
      "<gensim.matutils.Dense2Corpus object at 0x0000000023B57978>\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[ 0.,  0.],\n",
       "       [ 1.,  4.],\n",
       "       [ 6.,  8.],\n",
       "       [ 6.,  9.],\n",
       "       [ 5.,  8.]], dtype=float32)"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "numpy_matrix = np.random.randint(10, size=[5,2])  # random matrix as an example\n",
    "print(numpy_matrix)\n",
    "corpus = gensim.matutils.Dense2Corpus(numpy_matrix)\n",
    "print(corpus)\n",
    "numpy_matrix = gensim.matutils.corpus2dense(corpus, num_terms=5)\n",
    "numpy_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<HDF5 dataset \"tfidf_0\": shape (127930,), type \"<f4\">\n",
      "<HDF5 dataset \"tfidf_1\": shape (127930,), type \"<f4\">\n",
      "<HDF5 dataset \"tfidf_10\": shape (127930,), type \"<f4\">\n",
      "<HDF5 dataset \"tfidf_11\": shape (127930,), type \"<f4\">\n",
      "<HDF5 dataset \"tfidf_12\": shape (127930,), type \"<f4\">\n",
      "<HDF5 dataset \"tfidf_13\": shape (127930,), type \"<f4\">\n",
      "<HDF5 dataset \"tfidf_14\": shape (127930,), type \"<f4\">\n",
      "<HDF5 dataset \"tfidf_15\": shape (127930,), type \"<f4\">\n",
      "<HDF5 dataset \"tfidf_16\": shape (127930,), type \"<f4\">\n",
      "<HDF5 dataset \"tfidf_17\": shape (127930,), type \"<f4\">\n",
      "<HDF5 dataset \"tfidf_18\": shape (127930,), type \"<f4\">\n",
      "<HDF5 dataset \"tfidf_19\": shape (127930,), type \"<f4\">\n",
      "<HDF5 dataset \"tfidf_2\": shape (127930,), type \"<f4\">\n",
      "<HDF5 dataset \"tfidf_20\": shape (127930,), type \"<f4\">\n",
      "<HDF5 dataset \"tfidf_21\": shape (127930,), type \"<f4\">\n",
      "<HDF5 dataset \"tfidf_22\": shape (127930,), type \"<f4\">\n",
      "<HDF5 dataset \"tfidf_23\": shape (127930,), type \"<f4\">\n",
      "<HDF5 dataset \"tfidf_24\": shape (127930,), type \"<f4\">\n",
      "<HDF5 dataset \"tfidf_25\": shape (127930,), type \"<f4\">\n",
      "<HDF5 dataset \"tfidf_26\": shape (127930,), type \"<f4\">\n",
      "<HDF5 dataset \"tfidf_27\": shape (127930,), type \"<f4\">\n",
      "<HDF5 dataset \"tfidf_28\": shape (127930,), type \"<f4\">\n",
      "<HDF5 dataset \"tfidf_29\": shape (127930,), type \"<f4\">\n",
      "<HDF5 dataset \"tfidf_3\": shape (127930,), type \"<f4\">\n",
      "<HDF5 dataset \"tfidf_4\": shape (127930,), type \"<f4\">\n",
      "<HDF5 dataset \"tfidf_5\": shape (127930,), type \"<f4\">\n",
      "<HDF5 dataset \"tfidf_6\": shape (127930,), type \"<f4\">\n",
      "<HDF5 dataset \"tfidf_7\": shape (127930,), type \"<f4\">\n",
      "<HDF5 dataset \"tfidf_8\": shape (127930,), type \"<f4\">\n",
      "<HDF5 dataset \"tfidf_9\": shape (127930,), type \"<f4\">\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<HDF5 dataset \"tfidf_2\": shape (127930,), type \"<f4\">"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file = 'tfidf.hdf5'\n",
    "check = h5py.File(os.path.join(root,os.path.join('tmp',file)), 'r')\n",
    "for n in check:\n",
    "    print(check[str(n)])\n",
    "check['tfidf_2']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "10\n",
      "20\n",
      "30\n",
      "40\n",
      "50\n",
      "60\n",
      "70\n",
      "80\n",
      "90\n"
     ]
    }
   ],
   "source": [
    "test = range(0,100)\n",
    "for n in test:\n",
    "    if n%10 == 0:\n",
    "        print(n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "all the input array dimensions except for the concatenation axis must match exactly",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-106-6b301e3690b9>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[1;31m# DataSampled['sim'] = simsss\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mDataSampled\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'sim'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msimsss\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mX\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvstack\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mDataSampled\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msim\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0mkmeans\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mMiniBatchKMeans\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mn_clusters\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m90\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch_size\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m100\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mpred_classes\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mkmeans\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mC:\\Users\\ifue3702\\AppData\\Local\\Continuum\\Anaconda3\\lib\\site-packages\\numpy\\core\\shape_base.py\u001b[0m in \u001b[0;36mvstack\u001b[0;34m(tup)\u001b[0m\n\u001b[1;32m    235\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m    236\u001b[0m     \"\"\"\n\u001b[0;32m--> 237\u001b[0;31m     \u001b[1;32mreturn\u001b[0m \u001b[0m_nx\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mconcatenate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0matleast_2d\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0m_m\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0m_m\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mtup\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    238\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m    239\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mhstack\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtup\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: all the input array dimensions except for the concatenation axis must match exactly"
     ]
    }
   ],
   "source": [
    "# DataSampled['sim'] = simsss\n",
    "DataSampled['vecs'] = vectors\n",
    "X = np.vstack(DataSampled.vecs)\n",
    "kmeans = MiniBatchKMeans(n_clusters = 90, batch_size = 100).fit(X)\n",
    "pred_classes = kmeans.predict(X)\n",
    "\n",
    "print(metrics.adjusted_rand_score(DataSampled.OWN.tolist(), pred_classes.tolist()),\n",
    "      metrics.adjusted_mutual_info_score(DataSampled.OWN.tolist(), pred_classes.tolist()), \n",
    "      metrics.completeness_score(DataSampled.OWN.tolist(), pred_classes.tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:gensim.models.lsimodel:using serial LSI version on this node\n",
      "INFO:gensim.models.lsimodel:updating model with new documents\n",
      "INFO:gensim.models.lsimodel:preparing a new chunk of documents\n",
      "INFO:gensim.models.lsimodel:using 100 extra samples and 2 power iterations\n",
      "INFO:gensim.models.lsimodel:1st phase: constructing (1205, 400) action matrix\n",
      "INFO:gensim.models.lsimodel:orthonormalizing (1205, 400) action matrix\n",
      "INFO:gensim.models.lsimodel:2nd phase: running dense svd on (400, 20000) matrix\n",
      "INFO:gensim.models.lsimodel:computing the final decomposition\n",
      "INFO:gensim.models.lsimodel:keeping 240 factors (discarding 0.000% of energy spectrum)\n",
      "INFO:gensim.models.lsimodel:processed documents up to #20000\n",
      "INFO:gensim.models.lsimodel:topic #0(52.220): 0.936*\"clay\" + 0.194*\"gravel\" + 0.191*\"sandy\" + 0.115*\"supply\" + 0.109*\"water\" + 0.091*\"yellow\" + 0.074*\"shale\" + 0.072*\"sand\" + 0.051*\"sandstone\" + 0.030*\"grey\"\n",
      "INFO:gensim.models.lsimodel:topic #1(44.171): -0.675*\"shale\" + -0.418*\"supply\" + -0.382*\"water\" + -0.307*\"sandstone\" + 0.231*\"clay\" + -0.140*\"sand\" + -0.127*\"grey\" + -0.113*\"gravel\" + -0.086*\"hard\" + -0.081*\"sandy\"\n",
      "INFO:gensim.models.lsimodel:topic #2(41.369): -0.683*\"shale\" + 0.412*\"supply\" + 0.378*\"water\" + 0.309*\"sandstone\" + 0.203*\"gravel\" + 0.194*\"sand\" + -0.130*\"sandy\" + -0.109*\"grey\" + -0.089*\"clay\" + 0.046*\"bearing\"\n",
      "INFO:gensim.models.lsimodel:topic #3(35.944): 0.840*\"sandstone\" + -0.298*\"gravel\" + -0.249*\"sand\" + -0.214*\"supply\" + -0.206*\"water\" + 0.176*\"yellow\" + 0.089*\"hard\" + 0.064*\"clay\" + 0.053*\"white\" + 0.049*\"grey\"\n",
      "INFO:gensim.models.lsimodel:topic #4(30.712): -0.878*\"sandy\" + 0.293*\"gravel\" + 0.175*\"shale\" + -0.146*\"soil\" + 0.133*\"clay\" + -0.131*\"yellow\" + -0.121*\"supply\" + 0.118*\"sandstone\" + -0.110*\"water\" + 0.095*\"sand\"\n",
      "INFO:gensim.models.lsimodel:preparing a new chunk of documents\n",
      "INFO:gensim.models.lsimodel:using 100 extra samples and 2 power iterations\n",
      "INFO:gensim.models.lsimodel:1st phase: constructing (1205, 400) action matrix\n",
      "INFO:gensim.models.lsimodel:orthonormalizing (1205, 400) action matrix\n",
      "INFO:gensim.models.lsimodel:2nd phase: running dense svd on (400, 20000) matrix\n",
      "INFO:gensim.models.lsimodel:computing the final decomposition\n",
      "INFO:gensim.models.lsimodel:keeping 300 factors (discarding 0.358% of energy spectrum)\n",
      "INFO:gensim.models.lsimodel:merging projections: (1205, 240) + (1205, 300)\n",
      "INFO:gensim.models.lsimodel:keeping 300 factors (discarding 0.196% of energy spectrum)\n",
      "INFO:gensim.models.lsimodel:processed documents up to #40000\n",
      "INFO:gensim.models.lsimodel:topic #0(78.335): 0.914*\"clay\" + 0.252*\"sandy\" + 0.248*\"gravel\" + 0.096*\"supply\" + 0.093*\"water\" + 0.083*\"sand\" + 0.069*\"yellow\" + 0.061*\"grey\" + 0.037*\"fine\" + 0.028*\"shale\"\n",
      "INFO:gensim.models.lsimodel:topic #1(62.091): -0.541*\"supply\" + -0.508*\"water\" + -0.463*\"gravel\" + -0.339*\"sand\" + 0.269*\"clay\" + -0.107*\"sandstone\" + -0.099*\"fine\" + -0.083*\"coarse\" + -0.080*\"bearing\" + -0.077*\"shale\"\n",
      "INFO:gensim.models.lsimodel:topic #2(50.600): -0.692*\"sandy\" + -0.593*\"shale\" + 0.254*\"gravel\" + -0.185*\"grey\" + 0.169*\"clay\" + -0.101*\"sandstone\" + 0.087*\"sand\" + -0.072*\"supply\" + -0.071*\"yellow\" + -0.063*\"hard\"\n",
      "INFO:gensim.models.lsimodel:topic #3(46.494): -0.649*\"sandy\" + 0.551*\"shale\" + -0.293*\"gravel\" + 0.215*\"clay\" + 0.206*\"sandstone\" + 0.162*\"supply\" + -0.138*\"sand\" + 0.136*\"water\" + 0.114*\"grey\" + -0.091*\"fine\"\n",
      "INFO:gensim.models.lsimodel:topic #4(45.341): 0.573*\"gravel\" + 0.505*\"shale\" + -0.371*\"supply\" + -0.325*\"water\" + -0.292*\"sandstone\" + 0.147*\"sand\" + -0.140*\"sandy\" + 0.126*\"grey\" + 0.090*\"fine\" + -0.078*\"yellow\"\n",
      "INFO:gensim.models.lsimodel:preparing a new chunk of documents\n",
      "INFO:gensim.models.lsimodel:using 100 extra samples and 2 power iterations\n",
      "INFO:gensim.models.lsimodel:1st phase: constructing (1205, 400) action matrix\n",
      "INFO:gensim.models.lsimodel:orthonormalizing (1205, 400) action matrix\n",
      "INFO:gensim.models.lsimodel:2nd phase: running dense svd on (400, 20000) matrix\n",
      "INFO:gensim.models.lsimodel:computing the final decomposition\n",
      "INFO:gensim.models.lsimodel:keeping 300 factors (discarding 0.601% of energy spectrum)\n",
      "INFO:gensim.models.lsimodel:merging projections: (1205, 300) + (1205, 300)\n",
      "INFO:gensim.models.lsimodel:keeping 300 factors (discarding 0.478% of energy spectrum)\n",
      "INFO:gensim.models.lsimodel:processed documents up to #60000\n",
      "INFO:gensim.models.lsimodel:topic #0(93.440): 0.901*\"clay\" + 0.269*\"gravel\" + 0.247*\"sandy\" + 0.102*\"sand\" + 0.095*\"grey\" + 0.090*\"supply\" + 0.087*\"water\" + 0.067*\"fine\" + 0.060*\"yellow\" + 0.058*\"brown\"\n",
      "INFO:gensim.models.lsimodel:topic #1(72.989): 0.519*\"gravel\" + 0.483*\"supply\" + 0.450*\"water\" + 0.373*\"sand\" + -0.309*\"clay\" + 0.151*\"fine\" + 0.102*\"coarse\" + 0.069*\"sandstone\" + 0.066*\"bearing\" + 0.050*\"shale\"\n",
      "INFO:gensim.models.lsimodel:topic #2(58.149): 0.867*\"sandy\" + 0.301*\"shale\" + -0.241*\"clay\" + 0.196*\"grey\" + -0.168*\"gravel\" + 0.085*\"light\" + 0.078*\"fine\" + 0.075*\"brown\" + 0.058*\"yellow\" + 0.043*\"sandstone\"\n",
      "INFO:gensim.models.lsimodel:topic #3(55.108): 0.510*\"gravel\" + -0.495*\"supply\" + -0.438*\"water\" + 0.237*\"fine\" + 0.226*\"sand\" + -0.206*\"shale\" + -0.189*\"sandstone\" + 0.159*\"brown\" + -0.149*\"clay\" + 0.138*\"light\"\n",
      "INFO:gensim.models.lsimodel:topic #4(51.283): 0.771*\"shale\" + 0.410*\"grey\" + -0.384*\"sandy\" + -0.142*\"supply\" + -0.128*\"water\" + 0.116*\"light\" + 0.076*\"hard\" + 0.076*\"brown\" + 0.073*\"sand\" + 0.065*\"sandstone\"\n",
      "INFO:gensim.models.lsimodel:preparing a new chunk of documents\n",
      "INFO:gensim.models.lsimodel:using 100 extra samples and 2 power iterations\n",
      "INFO:gensim.models.lsimodel:1st phase: constructing (1205, 400) action matrix\n",
      "INFO:gensim.models.lsimodel:orthonormalizing (1205, 400) action matrix\n",
      "INFO:gensim.models.lsimodel:2nd phase: running dense svd on (400, 20000) matrix\n",
      "INFO:gensim.models.lsimodel:computing the final decomposition\n",
      "INFO:gensim.models.lsimodel:keeping 300 factors (discarding 0.103% of energy spectrum)\n",
      "INFO:gensim.models.lsimodel:merging projections: (1205, 300) + (1205, 300)\n",
      "INFO:gensim.models.lsimodel:keeping 300 factors (discarding 0.220% of energy spectrum)\n",
      "INFO:gensim.models.lsimodel:processed documents up to #80000\n",
      "INFO:gensim.models.lsimodel:topic #0(109.021): 0.919*\"clay\" + 0.241*\"gravel\" + 0.224*\"sandy\" + 0.092*\"sand\" + 0.091*\"grey\" + 0.084*\"supply\" + 0.082*\"water\" + 0.058*\"brown\" + 0.054*\"light\" + 0.053*\"fine\"\n",
      "INFO:gensim.models.lsimodel:topic #1(85.112): 0.510*\"supply\" + 0.502*\"gravel\" + 0.479*\"water\" + 0.364*\"sand\" + -0.276*\"clay\" + 0.125*\"fine\" + 0.087*\"coarse\" + 0.075*\"bearing\" + 0.065*\"sandstone\" + 0.065*\"shale\"\n",
      "INFO:gensim.models.lsimodel:topic #2(66.026): 0.853*\"sandy\" + 0.336*\"shale\" + 0.212*\"grey\" + -0.210*\"clay\" + -0.188*\"gravel\" + 0.091*\"light\" + 0.073*\"brown\" + 0.065*\"yellow\" + 0.063*\"fine\" + -0.049*\"sand\"\n",
      "INFO:gensim.models.lsimodel:topic #3(63.484): 0.526*\"gravel\" + -0.460*\"supply\" + -0.402*\"water\" + 0.268*\"sand\" + -0.255*\"shale\" + 0.213*\"fine\" + 0.172*\"sandy\" + 0.167*\"brown\" + -0.161*\"clay\" + 0.160*\"light\"\n",
      "INFO:gensim.models.lsimodel:topic #4(59.135): 0.761*\"shale\" + -0.398*\"sandy\" + 0.384*\"grey\" + -0.174*\"supply\" + -0.157*\"water\" + 0.127*\"light\" + 0.085*\"brown\" + 0.084*\"gravel\" + 0.079*\"sand\" + 0.078*\"hard\"\n",
      "INFO:gensim.models.lsimodel:preparing a new chunk of documents\n",
      "INFO:gensim.models.lsimodel:using 100 extra samples and 2 power iterations\n",
      "INFO:gensim.models.lsimodel:1st phase: constructing (1205, 400) action matrix\n",
      "INFO:gensim.models.lsimodel:orthonormalizing (1205, 400) action matrix\n",
      "INFO:gensim.models.lsimodel:2nd phase: running dense svd on (400, 20000) matrix\n",
      "INFO:gensim.models.lsimodel:computing the final decomposition\n",
      "INFO:gensim.models.lsimodel:keeping 300 factors (discarding 0.136% of energy spectrum)\n",
      "INFO:gensim.models.lsimodel:merging projections: (1205, 300) + (1205, 300)\n",
      "INFO:gensim.models.lsimodel:keeping 300 factors (discarding 0.326% of energy spectrum)\n",
      "INFO:gensim.models.lsimodel:processed documents up to #100000\n",
      "INFO:gensim.models.lsimodel:topic #0(118.494): 0.918*\"clay\" + 0.233*\"sandy\" + 0.232*\"gravel\" + 0.098*\"sand\" + 0.096*\"grey\" + 0.081*\"brown\" + 0.077*\"water\" + 0.073*\"supply\" + 0.053*\"light\" + 0.048*\"fine\"\n",
      "INFO:gensim.models.lsimodel:topic #1(92.103): 0.534*\"gravel\" + 0.456*\"water\" + 0.453*\"supply\" + 0.407*\"sand\" + -0.272*\"clay\" + 0.125*\"fine\" + 0.108*\"bearing\" + 0.093*\"coarse\" + 0.080*\"shale\" + 0.059*\"sandstone\"\n",
      "INFO:gensim.models.lsimodel:topic #2(72.863): 0.740*\"sandy\" + 0.495*\"shale\" + -0.258*\"gravel\" + 0.214*\"grey\" + -0.184*\"clay\" + 0.106*\"supply\" + -0.104*\"sand\" + 0.097*\"water\" + 0.080*\"brown\" + 0.078*\"yellow\"\n",
      "INFO:gensim.models.lsimodel:topic #3(69.718): 0.452*\"sandy\" + -0.403*\"supply\" + 0.370*\"gravel\" + -0.368*\"water\" + -0.361*\"shale\" + 0.275*\"sand\" + -0.197*\"clay\" + 0.173*\"fine\" + 0.156*\"brown\" + -0.129*\"sandstone\"\n",
      "INFO:gensim.models.lsimodel:topic #4(66.575): 0.665*\"shale\" + -0.400*\"sandy\" + -0.308*\"supply\" + -0.286*\"water\" + 0.278*\"grey\" + 0.186*\"brown\" + 0.178*\"sand\" + 0.175*\"gravel\" + 0.135*\"light\" + 0.087*\"fine\"\n",
      "INFO:gensim.models.lsimodel:preparing a new chunk of documents\n",
      "INFO:gensim.models.lsimodel:using 100 extra samples and 2 power iterations\n",
      "INFO:gensim.models.lsimodel:1st phase: constructing (1205, 400) action matrix\n",
      "INFO:gensim.models.lsimodel:orthonormalizing (1205, 400) action matrix\n",
      "INFO:gensim.models.lsimodel:2nd phase: running dense svd on (400, 20000) matrix\n",
      "INFO:gensim.models.lsimodel:computing the final decomposition\n",
      "INFO:gensim.models.lsimodel:keeping 300 factors (discarding 0.315% of energy spectrum)\n",
      "INFO:gensim.models.lsimodel:merging projections: (1205, 300) + (1205, 300)\n",
      "INFO:gensim.models.lsimodel:keeping 300 factors (discarding 0.363% of energy spectrum)\n",
      "INFO:gensim.models.lsimodel:processed documents up to #120000\n",
      "INFO:gensim.models.lsimodel:topic #0(127.730): 0.923*\"clay\" + 0.223*\"gravel\" + 0.215*\"sandy\" + 0.117*\"brown\" + 0.101*\"sand\" + 0.100*\"grey\" + 0.062*\"water\" + 0.056*\"supply\" + 0.050*\"light\" + 0.043*\"fine\"\n",
      "INFO:gensim.models.lsimodel:topic #1(98.383): 0.610*\"gravel\" + 0.473*\"sand\" + 0.379*\"water\" + 0.351*\"supply\" + -0.269*\"clay\" + 0.126*\"fine\" + 0.119*\"bearing\" + 0.094*\"coarse\" + 0.078*\"shale\" + 0.070*\"brown\"\n",
      "INFO:gensim.models.lsimodel:topic #2(79.321): 0.763*\"shale\" + 0.406*\"sandy\" + -0.273*\"gravel\" + 0.235*\"grey\" + 0.160*\"brown\" + 0.149*\"water\" + 0.140*\"supply\" + -0.114*\"clay\" + -0.110*\"sand\" + 0.079*\"sandstone\"\n",
      "INFO:gensim.models.lsimodel:topic #3(74.536): 0.668*\"sandy\" + -0.356*\"water\" + -0.346*\"supply\" + 0.231*\"sand\" + -0.219*\"clay\" + -0.217*\"shale\" + 0.207*\"brown\" + 0.163*\"gravel\" + 0.136*\"fine\" + 0.127*\"light\"\n",
      "INFO:gensim.models.lsimodel:topic #4(72.841): 0.546*\"sandy\" + -0.443*\"shale\" + 0.376*\"water\" + 0.367*\"supply\" + -0.261*\"brown\" + -0.231*\"sand\" + -0.165*\"gravel\" + -0.147*\"grey\" + 0.138*\"basalt\" + -0.103*\"light\"\n",
      "INFO:gensim.models.lsimodel:preparing a new chunk of documents\n",
      "INFO:gensim.models.lsimodel:using 100 extra samples and 2 power iterations\n",
      "INFO:gensim.models.lsimodel:1st phase: constructing (1205, 400) action matrix\n",
      "INFO:gensim.models.lsimodel:orthonormalizing (1205, 400) action matrix\n",
      "INFO:gensim.models.lsimodel:2nd phase: running dense svd on (400, 7930) matrix\n",
      "INFO:gensim.models.lsimodel:computing the final decomposition\n",
      "INFO:gensim.models.lsimodel:keeping 300 factors (discarding 0.814% of energy spectrum)\n",
      "INFO:gensim.models.lsimodel:merging projections: (1205, 300) + (1205, 300)\n",
      "INFO:gensim.models.lsimodel:keeping 300 factors (discarding 0.340% of energy spectrum)\n",
      "INFO:gensim.models.lsimodel:processed documents up to #127930\n",
      "INFO:gensim.models.lsimodel:topic #0(129.951): 0.924*\"clay\" + 0.219*\"gravel\" + 0.212*\"sandy\" + 0.124*\"brown\" + 0.101*\"sand\" + 0.100*\"grey\" + 0.059*\"water\" + 0.053*\"supply\" + 0.050*\"light\" + 0.043*\"fine\"\n",
      "INFO:gensim.models.lsimodel:topic #1(99.834): 0.615*\"gravel\" + 0.480*\"sand\" + 0.369*\"water\" + 0.333*\"supply\" + -0.269*\"clay\" + 0.128*\"bearing\" + 0.128*\"fine\" + 0.096*\"coarse\" + 0.083*\"shale\" + 0.077*\"brown\"\n",
      "INFO:gensim.models.lsimodel:topic #2(81.120): 0.788*\"shale\" + 0.350*\"sandy\" + -0.271*\"gravel\" + 0.235*\"grey\" + 0.198*\"brown\" + 0.135*\"water\" + 0.118*\"supply\" + -0.108*\"clay\" + -0.105*\"sand\" + 0.079*\"sandstone\"\n",
      "INFO:gensim.models.lsimodel:topic #3(75.793): -0.593*\"sandy\" + 0.397*\"water\" + 0.367*\"supply\" + -0.256*\"brown\" + -0.255*\"sand\" + 0.209*\"clay\" + 0.196*\"basalt\" + -0.154*\"gravel\" + 0.153*\"shale\" + -0.141*\"light\"\n",
      "INFO:gensim.models.lsimodel:topic #4(74.274): 0.658*\"sandy\" + -0.410*\"shale\" + 0.328*\"water\" + 0.305*\"supply\" + -0.241*\"brown\" + -0.200*\"sand\" + 0.183*\"basalt\" + -0.138*\"gravel\" + -0.106*\"grey\" + 0.106*\"bearing\"\n"
     ]
    }
   ],
   "source": [
    "lsi = gensim.models.LsiModel(corpus_tfidf, id2word=dictionary, num_topics=300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:gensim.similarities.docsim:starting similarity index under C:\\Users\\ifue3702\\Documents\\GitHub\\Litho/tmp/\n",
      "INFO:gensim.similarities.docsim:PROGRESS: fresh_shard size=10000\n",
      "INFO:gensim.similarities.docsim:PROGRESS: fresh_shard size=20000\n",
      "INFO:gensim.similarities.docsim:PROGRESS: fresh_shard size=30000\n",
      "INFO:gensim.similarities.docsim:creating sparse index\n",
      "INFO:gensim.matutils:creating sparse matrix from corpus\n",
      "INFO:gensim.matutils:PROGRESS: at document #0/32768\n",
      "INFO:gensim.matutils:PROGRESS: at document #10000/32768\n",
      "INFO:gensim.matutils:PROGRESS: at document #20000/32768\n",
      "INFO:gensim.matutils:PROGRESS: at document #30000/32768\n",
      "INFO:gensim.similarities.docsim:created <32768x1205 sparse matrix of type '<class 'numpy.float32'>'\n",
      "\twith 9826486 stored elements in Compressed Sparse Row format>\n",
      "INFO:gensim.similarities.docsim:creating sparse shard #0\n",
      "INFO:gensim.similarities.docsim:saving index shard to C:\\Users\\ifue3702\\Documents\\GitHub\\Litho/tmp\\.0\n",
      "INFO:gensim.utils:saving SparseMatrixSimilarity object under C:\\Users\\ifue3702\\Documents\\GitHub\\Litho/tmp\\.0, separately None\n",
      "INFO:gensim.utils:saved C:\\Users\\ifue3702\\Documents\\GitHub\\Litho/tmp\\.0\n",
      "INFO:gensim.utils:loading SparseMatrixSimilarity object from C:\\Users\\ifue3702\\Documents\\GitHub\\Litho/tmp\\.0\n",
      "INFO:gensim.utils:loaded C:\\Users\\ifue3702\\Documents\\GitHub\\Litho/tmp\\.0\n",
      "INFO:gensim.similarities.docsim:PROGRESS: fresh_shard size=0\n",
      "INFO:gensim.similarities.docsim:PROGRESS: fresh_shard size=10000\n",
      "INFO:gensim.similarities.docsim:PROGRESS: fresh_shard size=20000\n",
      "INFO:gensim.similarities.docsim:PROGRESS: fresh_shard size=30000\n",
      "INFO:gensim.similarities.docsim:creating sparse index\n",
      "INFO:gensim.matutils:creating sparse matrix from corpus\n",
      "INFO:gensim.matutils:PROGRESS: at document #0/32768\n",
      "INFO:gensim.matutils:PROGRESS: at document #10000/32768\n",
      "INFO:gensim.matutils:PROGRESS: at document #20000/32768\n",
      "INFO:gensim.matutils:PROGRESS: at document #30000/32768\n",
      "INFO:gensim.similarities.docsim:created <32768x1205 sparse matrix of type '<class 'numpy.float32'>'\n",
      "\twith 9810852 stored elements in Compressed Sparse Row format>\n",
      "INFO:gensim.similarities.docsim:creating sparse shard #1\n",
      "INFO:gensim.similarities.docsim:saving index shard to C:\\Users\\ifue3702\\Documents\\GitHub\\Litho/tmp\\.1\n",
      "INFO:gensim.utils:saving SparseMatrixSimilarity object under C:\\Users\\ifue3702\\Documents\\GitHub\\Litho/tmp\\.1, separately None\n",
      "INFO:gensim.utils:saved C:\\Users\\ifue3702\\Documents\\GitHub\\Litho/tmp\\.1\n",
      "INFO:gensim.utils:loading SparseMatrixSimilarity object from C:\\Users\\ifue3702\\Documents\\GitHub\\Litho/tmp\\.1\n",
      "INFO:gensim.utils:loaded C:\\Users\\ifue3702\\Documents\\GitHub\\Litho/tmp\\.1\n",
      "INFO:gensim.similarities.docsim:PROGRESS: fresh_shard size=0\n",
      "INFO:gensim.similarities.docsim:PROGRESS: fresh_shard size=10000\n",
      "INFO:gensim.similarities.docsim:PROGRESS: fresh_shard size=20000\n",
      "INFO:gensim.similarities.docsim:PROGRESS: fresh_shard size=30000\n",
      "INFO:gensim.similarities.docsim:creating sparse index\n",
      "INFO:gensim.matutils:creating sparse matrix from corpus\n",
      "INFO:gensim.matutils:PROGRESS: at document #0/32768\n",
      "INFO:gensim.matutils:PROGRESS: at document #10000/32768\n",
      "INFO:gensim.matutils:PROGRESS: at document #20000/32768\n",
      "INFO:gensim.matutils:PROGRESS: at document #30000/32768\n",
      "INFO:gensim.similarities.docsim:created <32768x1205 sparse matrix of type '<class 'numpy.float32'>'\n",
      "\twith 9827978 stored elements in Compressed Sparse Row format>\n",
      "INFO:gensim.similarities.docsim:creating sparse shard #2\n",
      "INFO:gensim.similarities.docsim:saving index shard to C:\\Users\\ifue3702\\Documents\\GitHub\\Litho/tmp\\.2\n",
      "INFO:gensim.utils:saving SparseMatrixSimilarity object under C:\\Users\\ifue3702\\Documents\\GitHub\\Litho/tmp\\.2, separately None\n",
      "INFO:gensim.utils:saved C:\\Users\\ifue3702\\Documents\\GitHub\\Litho/tmp\\.2\n",
      "INFO:gensim.utils:loading SparseMatrixSimilarity object from C:\\Users\\ifue3702\\Documents\\GitHub\\Litho/tmp\\.2\n",
      "INFO:gensim.utils:loaded C:\\Users\\ifue3702\\Documents\\GitHub\\Litho/tmp\\.2\n",
      "INFO:gensim.similarities.docsim:PROGRESS: fresh_shard size=0\n",
      "INFO:gensim.similarities.docsim:PROGRESS: fresh_shard size=10000\n",
      "INFO:gensim.similarities.docsim:PROGRESS: fresh_shard size=20000\n",
      "INFO:gensim.similarities.docsim:creating sparse index\n",
      "INFO:gensim.matutils:creating sparse matrix from corpus\n",
      "INFO:gensim.matutils:PROGRESS: at document #0/29626\n",
      "INFO:gensim.matutils:PROGRESS: at document #10000/29626\n",
      "INFO:gensim.matutils:PROGRESS: at document #20000/29626\n",
      "INFO:gensim.similarities.docsim:created <29626x1205 sparse matrix of type '<class 'numpy.float32'>'\n",
      "\twith 8880512 stored elements in Compressed Sparse Row format>\n",
      "INFO:gensim.similarities.docsim:creating sparse shard #3\n",
      "INFO:gensim.similarities.docsim:saving index shard to C:\\Users\\ifue3702\\Documents\\GitHub\\Litho/tmp\\.3\n",
      "INFO:gensim.utils:saving SparseMatrixSimilarity object under C:\\Users\\ifue3702\\Documents\\GitHub\\Litho/tmp\\.3, separately None\n",
      "INFO:gensim.utils:saved C:\\Users\\ifue3702\\Documents\\GitHub\\Litho/tmp\\.3\n",
      "INFO:gensim.utils:loading SparseMatrixSimilarity object from C:\\Users\\ifue3702\\Documents\\GitHub\\Litho/tmp\\.3\n",
      "INFO:gensim.utils:loaded C:\\Users\\ifue3702\\Documents\\GitHub\\Litho/tmp\\.3\n"
     ]
    }
   ],
   "source": [
    "lsiSim = gensim.similarities.Similarity(root+'/tmp/',lsi[corpus_tfidf], len(dictionary))\n",
    "# vectors =[]\n",
    "simsss = []\n",
    "for n in lemmatized[:10]:\n",
    "    vec_bow = dictionary.doc2bow(n)\n",
    "    vec_lsi = lsi[vec_bow]\n",
    "    sims1 = lsiSim[vec_lsi]\n",
    "    simsss.append(sims1)\n",
    "    \n",
    "#     vecs.append(vec_lsi)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:bokeh.core.state:Session output file 'plot.html' already exists, will be overwritten.\n"
     ]
    }
   ],
   "source": [
    "plot_embedding([x for x in simsss[:1000]],\n",
    "               [';'.join(x) for x in lemmatized[:1000]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "lda = gensim.models.LdaMulticore(corpus_tfidf, id2word=dictionary, num_topics=50, chunksize=10000, passes=10, workers = 8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:gensim.similarities.docsim:starting similarity index under C:\\Users\\ifue3702\\Documents\\GitHub\\Litho/tmp/\n",
      "INFO:gensim.similarities.docsim:PROGRESS: fresh_shard size=10000\n",
      "INFO:gensim.similarities.docsim:PROGRESS: fresh_shard size=20000\n",
      "INFO:gensim.similarities.docsim:PROGRESS: fresh_shard size=30000\n",
      "INFO:gensim.similarities.docsim:creating sparse index\n",
      "INFO:gensim.matutils:creating sparse matrix from corpus\n",
      "INFO:gensim.matutils:PROGRESS: at document #0/32768\n",
      "INFO:gensim.matutils:PROGRESS: at document #10000/32768\n",
      "INFO:gensim.matutils:PROGRESS: at document #20000/32768\n",
      "INFO:gensim.matutils:PROGRESS: at document #30000/32768\n",
      "INFO:gensim.similarities.docsim:created <32768x1205 sparse matrix of type '<class 'numpy.float32'>'\n",
      "\twith 57966 stored elements in Compressed Sparse Row format>\n",
      "INFO:gensim.similarities.docsim:creating sparse shard #0\n",
      "INFO:gensim.similarities.docsim:saving index shard to C:\\Users\\ifue3702\\Documents\\GitHub\\Litho/tmp\\.0\n",
      "INFO:gensim.utils:saving SparseMatrixSimilarity object under C:\\Users\\ifue3702\\Documents\\GitHub\\Litho/tmp\\.0, separately None\n",
      "INFO:gensim.utils:saved C:\\Users\\ifue3702\\Documents\\GitHub\\Litho/tmp\\.0\n",
      "INFO:gensim.utils:loading SparseMatrixSimilarity object from C:\\Users\\ifue3702\\Documents\\GitHub\\Litho/tmp\\.0\n",
      "INFO:gensim.utils:loaded C:\\Users\\ifue3702\\Documents\\GitHub\\Litho/tmp\\.0\n",
      "INFO:gensim.similarities.docsim:PROGRESS: fresh_shard size=0\n",
      "INFO:gensim.similarities.docsim:PROGRESS: fresh_shard size=10000\n",
      "INFO:gensim.similarities.docsim:PROGRESS: fresh_shard size=20000\n",
      "INFO:gensim.similarities.docsim:PROGRESS: fresh_shard size=30000\n",
      "INFO:gensim.similarities.docsim:creating sparse index\n",
      "INFO:gensim.matutils:creating sparse matrix from corpus\n",
      "INFO:gensim.matutils:PROGRESS: at document #0/32768\n",
      "INFO:gensim.matutils:PROGRESS: at document #10000/32768\n",
      "INFO:gensim.matutils:PROGRESS: at document #20000/32768\n",
      "INFO:gensim.matutils:PROGRESS: at document #30000/32768\n",
      "INFO:gensim.similarities.docsim:created <32768x1205 sparse matrix of type '<class 'numpy.float32'>'\n",
      "\twith 65624 stored elements in Compressed Sparse Row format>\n",
      "INFO:gensim.similarities.docsim:creating sparse shard #1\n",
      "INFO:gensim.similarities.docsim:saving index shard to C:\\Users\\ifue3702\\Documents\\GitHub\\Litho/tmp\\.1\n",
      "INFO:gensim.utils:saving SparseMatrixSimilarity object under C:\\Users\\ifue3702\\Documents\\GitHub\\Litho/tmp\\.1, separately None\n",
      "INFO:gensim.utils:saved C:\\Users\\ifue3702\\Documents\\GitHub\\Litho/tmp\\.1\n",
      "INFO:gensim.utils:loading SparseMatrixSimilarity object from C:\\Users\\ifue3702\\Documents\\GitHub\\Litho/tmp\\.1\n",
      "INFO:gensim.utils:loaded C:\\Users\\ifue3702\\Documents\\GitHub\\Litho/tmp\\.1\n",
      "INFO:gensim.similarities.docsim:PROGRESS: fresh_shard size=0\n",
      "INFO:gensim.similarities.docsim:PROGRESS: fresh_shard size=10000\n",
      "INFO:gensim.similarities.docsim:PROGRESS: fresh_shard size=20000\n",
      "INFO:gensim.similarities.docsim:PROGRESS: fresh_shard size=30000\n",
      "INFO:gensim.similarities.docsim:creating sparse index\n",
      "INFO:gensim.matutils:creating sparse matrix from corpus\n",
      "INFO:gensim.matutils:PROGRESS: at document #0/32768\n",
      "INFO:gensim.matutils:PROGRESS: at document #10000/32768\n",
      "INFO:gensim.matutils:PROGRESS: at document #20000/32768\n",
      "INFO:gensim.matutils:PROGRESS: at document #30000/32768\n",
      "INFO:gensim.similarities.docsim:created <32768x1205 sparse matrix of type '<class 'numpy.float32'>'\n",
      "\twith 52279 stored elements in Compressed Sparse Row format>\n",
      "INFO:gensim.similarities.docsim:creating sparse shard #2\n",
      "INFO:gensim.similarities.docsim:saving index shard to C:\\Users\\ifue3702\\Documents\\GitHub\\Litho/tmp\\.2\n",
      "INFO:gensim.utils:saving SparseMatrixSimilarity object under C:\\Users\\ifue3702\\Documents\\GitHub\\Litho/tmp\\.2, separately None\n",
      "INFO:gensim.utils:saved C:\\Users\\ifue3702\\Documents\\GitHub\\Litho/tmp\\.2\n",
      "INFO:gensim.utils:loading SparseMatrixSimilarity object from C:\\Users\\ifue3702\\Documents\\GitHub\\Litho/tmp\\.2\n",
      "INFO:gensim.utils:loaded C:\\Users\\ifue3702\\Documents\\GitHub\\Litho/tmp\\.2\n",
      "INFO:gensim.similarities.docsim:PROGRESS: fresh_shard size=0\n",
      "INFO:gensim.similarities.docsim:PROGRESS: fresh_shard size=10000\n",
      "INFO:gensim.similarities.docsim:PROGRESS: fresh_shard size=20000\n",
      "INFO:gensim.similarities.docsim:creating sparse index\n",
      "INFO:gensim.matutils:creating sparse matrix from corpus\n",
      "INFO:gensim.matutils:PROGRESS: at document #0/29626\n",
      "INFO:gensim.matutils:PROGRESS: at document #10000/29626\n",
      "INFO:gensim.matutils:PROGRESS: at document #20000/29626\n",
      "INFO:gensim.similarities.docsim:created <29626x1205 sparse matrix of type '<class 'numpy.float32'>'\n",
      "\twith 49375 stored elements in Compressed Sparse Row format>\n",
      "INFO:gensim.similarities.docsim:creating sparse shard #3\n",
      "INFO:gensim.similarities.docsim:saving index shard to C:\\Users\\ifue3702\\Documents\\GitHub\\Litho/tmp\\.3\n",
      "INFO:gensim.utils:saving SparseMatrixSimilarity object under C:\\Users\\ifue3702\\Documents\\GitHub\\Litho/tmp\\.3, separately None\n",
      "INFO:gensim.utils:saved C:\\Users\\ifue3702\\Documents\\GitHub\\Litho/tmp\\.3\n",
      "INFO:gensim.utils:loading SparseMatrixSimilarity object from C:\\Users\\ifue3702\\Documents\\GitHub\\Litho/tmp\\.3\n",
      "INFO:gensim.utils:loaded C:\\Users\\ifue3702\\Documents\\GitHub\\Litho/tmp\\.3\n"
     ]
    }
   ],
   "source": [
    "ldaSim = gensim.similarities.Similarity(root+'/tmp/', lda[corpus], len(dictionary))\n",
    "# vectors =[]\n",
    "simsss = []\n",
    "for n in lemmatized:\n",
    "    vec_bow = dictionary.doc2bow(n)\n",
    "    vec_lda = lda[vec_bow]\n",
    "    sims1 = ldaSim[vec_lda]\n",
    "    simsss.append(sims1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 408,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "size = 1000\n",
    "labs = list(range(len(totalvocab_lemmatized)))\n",
    "labs = ['lab_{}'.format(n) for n in labs]\n",
    "taggedDescriptions = labeledSentences(totalvocab_lemmatized, labels = labs)\n",
    "doc2vec = gensim.models.Doc2Vec(size = size, window = 13, workers = 8, seed = 1, \n",
    "                                min_count = 5, sample = 0.1e-5, dm = 1, dm_concat = 1, \n",
    "                                dm_mean = 1, alpha = 0.25)\n",
    "doc2vec.build_vocab(taggedDescriptions)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 409,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.434457623617 0.608323503602 0.665401278107\n"
     ]
    }
   ],
   "source": [
    "vectors = []\n",
    "for n in lemmatized:\n",
    "    vectors.append(np.reshape(doc2vec.infer_vector(n), (1,size)))\n",
    "vectors = np.reshape(np.asarray(vectors), (1000, size))\n",
    "kmeans = MiniBatchKMeans(n_clusters = 90, batch_size = 100).fit(vectors)\n",
    "pred_classes = kmeans.predict(vectors)\n",
    "\n",
    "print(metrics.adjusted_rand_score(DataSampled.OWN.tolist(), pred_classes.tolist()),\n",
    "      metrics.adjusted_mutual_info_score(DataSampled.OWN.tolist(), pred_classes.tolist()),\n",
    "      metrics.completeness_score(DataSampled.OWN.tolist(), pred_classes.tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from nltk.corpus import wordnet as wn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 233,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Synset('earth.n.02')\n",
      "the loose soft material that makes up a large part of the land surface\n"
     ]
    }
   ],
   "source": [
    "clay = wn.synset('clay.n.01')\n",
    "mud = wn.synset('mud.n.01')\n",
    "# pug = wn.synset('pug.n.02')\n",
    "sand = wn.synset('sand.n.01')\n",
    "silt = wn.synset('silt.n.01')\n",
    "gravel =  wn.synset('gravel.n.01')\n",
    "stone = wn.synset('stone.n.01')\n",
    "pebble = wn.synset('pebble.n.01')\n",
    "boulder = wn.synset('boulder.n.01')\n",
    "granite = wn.synset('granite.n.01')\n",
    "basalt = wn.synset('basalt.n.01')\n",
    "lava = wn.synset('lava.n.01')\n",
    "tuff = wn.synset('tuff.n.01')\n",
    "volcanic = wn.synset('volcanic.s.03')\n",
    "rhyolite = wn.synset('rhyolite.n.01')\n",
    "breccia = wn.synsets('breccia')[0]\n",
    "agglomerate = wn.synset('agglomerate.n.01')\n",
    "zeolite = wn.synset('zeolite.n.01')\n",
    "andesite = wn.synset('andesite.n.01')\n",
    "diorite = wn.synset('diorite.n.01')\n",
    "porphyry = wn.synset('porphyry.n.01')\n",
    "# dolerite = wn.synset('dolerite.n.01')\n",
    "feldspar = wn.synset('feldspar.n.01')\n",
    "igneous = wn.synset('igneous.a.01')\n",
    "\n",
    "sandstone = wn.synset('sandstone.n.01')\n",
    "# greywacke = wn.synset('greywacke.n.01')\n",
    "# arkose = wn.synset('arkose.n.01')\n",
    "\n",
    "shale = wn.synset('shale.n.01')\n",
    "claystone = wn.synset('claystone.n.01')\n",
    "# mudstone = wn.synset('mudstone.n.01')\n",
    "siltstone = wn.synset('siltstone.n.01')\n",
    "\n",
    "limestone =  wn.synset('limestone.n.01')\n",
    "dolomite = wn.synset('dolomite.n.01')\n",
    "# calcrete = wn.synset('calcrete.n.01')\n",
    "siderite = wn.synset('siderite.n.01')\n",
    "chalk = wn.synset('chalk.n.01')\n",
    "marl = wn.synset('marl.n.01')\n",
    "calcite = wn.synset('calcite.n.01')\n",
    "\n",
    "mica = wn.synset('mica.n.01')\n",
    "schist = wn.synset('schist.n.01')\n",
    "# serpentine = wn.synset('serpentine.n.01')\n",
    "soapstone = wn.synset('soapstone.n.01')\n",
    "slate = wn.synset('slate.n.03')\n",
    "# phyllite  = wn.synset('phyllite.n.01')\n",
    "\n",
    "carbonaceous = wn.synset('carbonaceous.a.01')\n",
    "coal = wn.synset('coal.n.01')\n",
    "lignite = wn.synset('lignite.n.01')\n",
    "wood = wn.synset('wood.n.01')\n",
    "\n",
    "jasper = wn.synset('jasper.n.01')\n",
    "chert = wn.synset('chert.n.01')\n",
    "# silcrete = wn.synset('silcrete.n.01')\n",
    "laterite = wn.synset('laterite.n.01')\n",
    "# ironstone = wn.synset('ironstone.n.01')\n",
    "cement = wn.synset('cement.n.03')\n",
    "# blueMetal = wn.synset('wood.n.01')\n",
    "\n",
    "soil = wn.synset('soil.n.02')\n",
    "loam = wn.synset('loam.n.01')\n",
    "wood = wn.synset('wood.n.01')\n",
    "\n",
    "bentonite = wn.synset('bentonite.n.01')\n",
    "\n",
    "conglomerate = wn.synset('conglomerate.n.01')\n",
    "\n",
    "drift = wn.synset('drift.n.04')\n",
    "\n",
    "quartz = wn.synset('quartz.n.02')\n",
    "\n",
    "rock = wn.synset('rock.n.01')\n",
    "alluvium = wn.synset('alluvium.n.01')\n",
    "water = wn.synset('water.n.01')\n",
    "cavity = wn.synset('cavity.n.01')\n",
    "sedimentary = wn.synset('sedimentary.a.01')\n",
    "channel = wn.synset('channel.n.02')\n",
    "ground = wn.synset('ground.n.07')\n",
    "earth = wn.synset('earth.n.02')\n",
    "\n",
    "sandy = wn.synsets('sandy', pos = wn.ADJ)[1]\n",
    "clayey = wn.synsets('clayey', pos = wn.ADJ)[0]\n",
    "silty = wn.synsets('silty')[0]\n",
    "gravelly = wn.synset('gravelly.s.01')\n",
    "stony = wn.synsets('stony')[0]\n",
    "pebbly = wn.synsets('pebbly')[0]\n",
    "gritty = wn.synsets('gritty')[0]\n",
    "\n",
    "\n",
    "\n",
    "# for n in drift:\n",
    "#     print(n.definition())\n",
    "print(earth)\n",
    "print(earth.definition())\n",
    "# sandy = wngreywacke.synset('sandy.a.01')\n",
    "# for n in sandy:\n",
    "#     print(n.definition())\n",
    "# print(sandy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "carbonaceous\n"
     ]
    }
   ],
   "source": [
    "\n",
    "Lemma = nltk.WordNetLemmatizer()\n",
    "print(Lemma.lemmatize('carbonaceous'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 247,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "vect = TfidfVectorizer()\n",
    "tfidf = vect.fit_transform([n.lower() for n in Descriptions])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.404928867022 0.232049174226 0.401791987721\n"
     ]
    }
   ],
   "source": [
    "kmeans = MiniBatchKMeans(n_clusters = 90, batch_size = 100, ).fit(tfidf)\n",
    "pred_classes = kmeans.predict(tfidf)\n",
    "\n",
    "print(metrics.completeness_score(Data.OWN.tolist(), pred_classes.tolist()),\n",
    "      metrics.adjusted_rand_score(Data.OWN.tolist(), pred_classes.tolist()),\n",
    "      metrics.adjusted_mutual_info_score(Data.OWN.tolist(), pred_classes.tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 412,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from __future__ import division\n",
    "import nltk\n",
    "from nltk.corpus import wordnet as wn\n",
    "from nltk.corpus import brown\n",
    "import math\n",
    "import numpy as np\n",
    "import sys\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 413,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Parameters to the algorithm. Currently set to values that was reported\n",
    "# in the paper to produce \"best\" results.\n",
    "ALPHA = 0.2\n",
    "BETA = 0.45\n",
    "ETA = 0.4\n",
    "PHI = 0.2\n",
    "DELTA = 0.85\n",
    "\n",
    "brown_freqs = dict()\n",
    "N = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 450,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "######################### word similarity ##########################\n",
    "\n",
    "def get_best_synset_pair(word_1, word_2):\n",
    "    \"\"\" \n",
    "    Choose the pair with highest path similarity among all pairs. \n",
    "    Mimics pattern-seeking behavior of humans.\n",
    "    \"\"\"\n",
    "    max_sim = -1.0\n",
    "    synsets_1 = wn.synsets(word_1)\n",
    "    synsets_2 = wn.synsets(word_2)\n",
    "    if len(synsets_1) == 0 or len(synsets_2) == 0:\n",
    "        return None, None\n",
    "    else:\n",
    "        max_sim = -1.0\n",
    "        best_pair = None, None\n",
    "        for synset_1 in synsets_1:\n",
    "            for synset_2 in synsets_2:\n",
    "                sim = wn.path_similarity(synset_1, synset_2)\n",
    "                if sim is not None and sim > max_sim:\n",
    "                    if sim > max_sim:\n",
    "                        max_sim = sim\n",
    "                        best_pair = synset_1, synset_2\n",
    "        return best_pair\n",
    "\n",
    "def length_dist(synset_1, synset_2):\n",
    "    \"\"\"\n",
    "    Return a measure of the length of the shortest path in the semantic\n",
    "    ontology (Wordnet in our case as well as the paper's) between two\n",
    "    synsets.\n",
    "    \"\"\"\n",
    "    l_dist = sys.maxsize\n",
    "    if synset_1 is None or synset_2 is None:\n",
    "        return 0.0\n",
    "    if synset_1 == synset_2:\n",
    "        # if synset_1 and synset_2 are the same synset return 0\n",
    "        l_dist = 0.0\n",
    "    else:\n",
    "        wset_1 = set([str(x.name()) for x in synset_1.lemmas()])\n",
    "        wset_2 = set([str(x.name()) for x in synset_2.lemmas()])\n",
    "        if len(wset_1.intersection(wset_2)) > 0:\n",
    "            # if synset_1 != synset_2 but there is word overlap, return 1.0\n",
    "            l_dist = 1.0\n",
    "        else:\n",
    "            # just compute the shortest path between the two\n",
    "            l_dist = synset_1.shortest_path_distance(synset_2)\n",
    "            if l_dist is None:\n",
    "                l_dist = 0.0\n",
    "    # normalize path length to the range [0,1]\n",
    "    return math.exp(-ALPHA * l_dist) \n",
    "\n",
    "def hierarchy_dist(synset_1, synset_2):\n",
    "    \"\"\"\n",
    "    Return a measure of depth in the ontology to model the fact that \n",
    "    nodes closer to the root are broader and have less semantic similarity\n",
    "    than nodes further away from the root.\n",
    "    \"\"\"\n",
    "    h_dist = sys.maxsize\n",
    "    if synset_1 is None or synset_2 is None: \n",
    "        return h_dist\n",
    "    if synset_1 == synset_2:\n",
    "        # return the depth of one of synset_1 or synset_2\n",
    "        h_dist = max([x[1] for x in synset_1.hypernym_distances()])\n",
    "    else:\n",
    "        # find the max depth of least common subsumer\n",
    "        hypernyms_1 = {x[0]:x[1] for x in synset_1.hypernym_distances()}\n",
    "        hypernyms_2 = {x[0]:x[1] for x in synset_2.hypernym_distances()}\n",
    "        lcs_candidates = set(hypernyms_1.keys()).intersection(\n",
    "            set(hypernyms_2.keys()))\n",
    "        if len(lcs_candidates) > 0:\n",
    "            lcs_dists = []\n",
    "            for lcs_candidate in lcs_candidates:\n",
    "                lcs_d1 = 0\n",
    "                if lcs_candidate in hypernyms_1:\n",
    "                    lcs_d1 = hypernyms_1[lcs_candidate]\n",
    "                lcs_d2 = 0\n",
    "                if lcs_candidate in hypernyms_2:\n",
    "                    lcs_d2 = hypernyms_2[lcs_candidate]\n",
    "                lcs_dists.append(max([lcs_d1, lcs_d2]))\n",
    "            h_dist = max(lcs_dists)\n",
    "        else:\n",
    "            h_dist = 0\n",
    "    return ((math.exp(BETA * h_dist) - math.exp(-BETA * h_dist)) / \n",
    "        (math.exp(BETA * h_dist) + math.exp(-BETA * h_dist)))\n",
    "    \n",
    "def word_similarity(word_1, word_2):\n",
    "    synset_pair = get_best_synset_pair(word_1, word_2)\n",
    "    return (length_dist(synset_pair[0], synset_pair[1]) * \n",
    "        hierarchy_dist(synset_pair[0], synset_pair[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 452,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "######################### sentence similarity ##########################\n",
    "\n",
    "def most_similar_word(word, word_set):\n",
    "    \"\"\"\n",
    "    Find the word in the joint word set that is most similar to the word\n",
    "    passed in. We use the algorithm above to compute word similarity between\n",
    "    the word and each word in the joint word set, and return the most similar\n",
    "    word and the actual similarity value.\n",
    "    \"\"\"\n",
    "    max_sim = -1.0\n",
    "    sim_word = \"\"\n",
    "    for ref_word in word_set:\n",
    "        sim = word_similarity(word, ref_word)\n",
    "        if sim > max_sim:\n",
    "            max_sim = sim\n",
    "            sim_word = ref_word\n",
    "        return sim_word, max_sim\n",
    "    \n",
    "def info_content(lookup_word):\n",
    "    \"\"\"\n",
    "    Uses the Brown corpus available in NLTK to calculate a Laplace\n",
    "    smoothed frequency distribution of words, then uses this information\n",
    "    to compute the information content of the lookup_word.\n",
    "    \"\"\"\n",
    "    global N\n",
    "    if N == 0:\n",
    "        # poor man's lazy evaluation\n",
    "        for sent in brown.sents():\n",
    "            for word in sent:\n",
    "                word = word.lower()\n",
    "                if word not in brown_freqs:\n",
    "                    brown_freqs[word] = 0\n",
    "                brown_freqs[word] = brown_freqs[word] + 1\n",
    "                N = N + 1\n",
    "    lookup_word = lookup_word.lower()\n",
    "    n = 0 if lookup_word not in brown_freqs else brown_freqs[lookup_word]\n",
    "    return 1.0 - (math.log(n + 1) / math.log(N + 1))\n",
    "    \n",
    "def semantic_vector(words, joint_words, info_content_norm):\n",
    "    \"\"\"\n",
    "    Computes the semantic vector of a sentence. The sentence is passed in as\n",
    "    a collection of words. The size of the semantic vector is the same as the\n",
    "    size of the joint word set. The elements are 1 if a word in the sentence\n",
    "    already exists in the joint word set, or the similarity of the word to the\n",
    "    most similar word in the joint word set if it doesn't. Both values are \n",
    "    further normalized by the word's (and similar word's) information content\n",
    "    if info_content_norm is True.\n",
    "    \"\"\"\n",
    "    sent_set = set(words)\n",
    "    semvec = np.zeros(len(joint_words))\n",
    "    i = 0\n",
    "    for joint_word in joint_words:\n",
    "        if joint_word in sent_set:\n",
    "            # if word in union exists in the sentence, s(i) = 1 (unnormalized)\n",
    "            semvec[i] = 1.0\n",
    "            if info_content_norm:\n",
    "                semvec[i] = semvec[i] * math.pow(info_content(joint_word), 2)\n",
    "        else:\n",
    "            # find the most similar word in the joint set and set the sim value\n",
    "            sim_word, max_sim = most_similar_word(joint_word, sent_set)\n",
    "            semvec[i] = PHI if max_sim > PHI else 0.0\n",
    "            if info_content_norm:\n",
    "                semvec[i] = semvec[i] * info_content(joint_word) * info_content(sim_word)\n",
    "        i = i + 1\n",
    "    return semvec                \n",
    "            \n",
    "def semantic_similarity(sentence_1, sentence_2, info_content_norm):\n",
    "    \"\"\"\n",
    "    Computes the semantic similarity between two sentences as the cosine\n",
    "    similarity between the semantic vectors computed for each sentence.\n",
    "    \"\"\"\n",
    "    words_1 = nltk.word_tokenize(sentence_1)\n",
    "    words_2 = nltk.word_tokenize(sentence_2)\n",
    "    joint_words = set(words_1).union(set(words_2))\n",
    "    vec_1 = semantic_vector(words_1, joint_words, info_content_norm)\n",
    "    vec_2 = semantic_vector(words_2, joint_words, info_content_norm)\n",
    "    return np.dot(vec_1, vec_2.T) / (np.linalg.norm(vec_1) * np.linalg.norm(vec_2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 416,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "######################### word order similarity ##########################\n",
    "\n",
    "def word_order_vector(words, joint_words, windex):\n",
    "    \"\"\"\n",
    "    Computes the word order vector for a sentence. The sentence is passed\n",
    "    in as a collection of words. The size of the word order vector is the\n",
    "    same as the size of the joint word set. The elements of the word order\n",
    "    vector are the position mapping (from the windex dictionary) of the \n",
    "    word in the joint set if the word exists in the sentence. If the word\n",
    "    does not exist in the sentence, then the value of the element is the \n",
    "    position of the most similar word in the sentence as long as the similarity\n",
    "    is above the threshold ETA.\n",
    "    \"\"\"\n",
    "    wovec = np.zeros(len(joint_words))\n",
    "    i = 0\n",
    "    wordset = set(words)\n",
    "    for joint_word in joint_words:\n",
    "        if joint_word in wordset:\n",
    "            # word in joint_words found in sentence, just populate the index\n",
    "            wovec[i] = windex[joint_word]\n",
    "        else:\n",
    "            # word not in joint_words, find most similar word and populate\n",
    "            # word_vector with the thresholded similarity\n",
    "            sim_word, max_sim = most_similar_word(joint_word, wordset)\n",
    "            if max_sim > ETA:\n",
    "                wovec[i] = windex[sim_word]\n",
    "            else:\n",
    "                wovec[i] = 0\n",
    "        i = i + 1\n",
    "    return wovec\n",
    "\n",
    "def word_order_similarity(sentence_1, sentence_2):\n",
    "    \"\"\"\n",
    "    Computes the word-order similarity between two sentences as the normalized\n",
    "    difference of word order between the two sentences.\n",
    "    \"\"\"\n",
    "    words_1 = nltk.word_tokenize(sentence_1)\n",
    "    words_2 = nltk.word_tokenize(sentence_2)\n",
    "    joint_words = list(set(words_1).union(set(words_2)))\n",
    "    windex = {x[1]: x[0] for x in enumerate(joint_words)}\n",
    "    r1 = word_order_vector(words_1, joint_words, windex)\n",
    "    r2 = word_order_vector(words_2, joint_words, windex)\n",
    "    return 1.0 - (np.linalg.norm(r1 - r2) / np.linalg.norm(r1 + r2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 417,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "######################### overall similarity ##########################\n",
    "\n",
    "def similarity(sentence_1, sentence_2, info_content_norm):\n",
    "    \"\"\"\n",
    "    Calculate the semantic similarity between two sentences. The last \n",
    "    parameter is True or False depending on whether information content\n",
    "    normalization is desired or not.\n",
    "    \"\"\"\n",
    "    return DELTA * semantic_similarity(sentence_1, sentence_2, info_content_norm) + \\\n",
    "        (1.0 - DELTA) * word_order_similarity(sentence_1, sentence_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 453,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I like that bachelor.\tI like that unmarried man.\t0.561\t0.788\t0.319\n",
      "John is very nice.\tIs John very nice?\t0.977\t0.615\t0.903\n",
      "Red alcoholic drink.\tA bottle of wine.\t0.585\t0.256\t0.069\n",
      "Red alcoholic drink.\tFresh orange juice.\t0.611\t0.268\t0.058\n",
      "Red alcoholic drink.\tAn English dictionary.\t0.000\t0.284\t0.074\n",
      "Red alcoholic drink.\tFresh apple juice.\t0.420\t0.252\t0.042\n",
      "A glass of cider.\tA full cup of apple juice.\t0.678\t0.626\t0.334\n",
      "It is a dog.\tThat must be your dog.\t0.739\t0.422\t0.682\n",
      "It is a dog.\tIt is a log.\t0.623\t0.808\t0.257\n",
      "It is a dog.\tIt is a pig.\t0.790\t0.784\t0.188\n",
      "Dogs are animals.\tThey are common pets.\t0.738\t0.420\t0.078\n",
      "Canis familiaris are animals.\tDogs are common pets.\t0.362\t0.394\t0.070\n",
      "I have a pen.\tWhere do you live?\t0.000\t0.101\t0.120\n",
      "I have a pen.\tWhere is ink?\t0.129\t0.100\t0.046\n",
      "I have a hammer.\tTake some nails.\t0.508\t0.256\t0.070\n",
      "I have a hammer.\tTake some apples.\t0.121\t0.293\t0.138\n"
     ]
    }
   ],
   "source": [
    "######################### main / test ##########################\n",
    "\n",
    "# the results of the algorithm are largely dependent on the results of \n",
    "# the word similarities, so we should test this first...\n",
    "# word_pairs = [\n",
    "#   [\"asylum\", \"fruit\", 0.21],\n",
    "#   [\"autograph\", \"shore\", 0.29],\n",
    "#   [\"autograph\", \"signature\", 0.55],\n",
    "#   [\"automobile\", \"car\", 0.64],\n",
    "#   [\"bird\", \"woodland\", 0.33],\n",
    "#   [\"boy\", \"rooster\", 0.53],\n",
    "#   [\"boy\", \"lad\", 0.66],\n",
    "#   [\"boy\", \"sage\", 0.51],\n",
    "#   [\"cemetery\", \"graveyard\", 0.73],\n",
    "#   [\"coast\", \"forest\", 0.36],\n",
    "#   [\"coast\", \"shore\", 0.76],\n",
    "#   [\"cock\", \"rooster\", 1.00],\n",
    "#   [\"cord\", \"smile\", 0.33],\n",
    "#   [\"cord\", \"string\", 0.68],\n",
    "#   [\"cushion\", \"pillow\", 0.66],\n",
    "#   [\"forest\", \"graveyard\", 0.55],\n",
    "#   [\"forest\", \"woodland\", 0.70],\n",
    "#   [\"furnace\", \"stove\", 0.72],\n",
    "#   [\"glass\", \"tumbler\", 0.65],\n",
    "#   [\"grin\", \"smile\", 0.49],\n",
    "#   [\"gem\", \"jewel\", 0.83],\n",
    "#   [\"hill\", \"woodland\", 0.59],\n",
    "#   [\"hill\", \"mound\", 0.74],\n",
    "#   [\"implement\", \"tool\", 0.75],\n",
    "#   [\"journey\", \"voyage\", 0.52],\n",
    "#   [\"magician\", \"oracle\", 0.44],\n",
    "#   [\"magician\", \"wizard\", 0.65],\n",
    "#   [\"midday\", \"noon\", 1.0],\n",
    "#   [\"oracle\", \"sage\", 0.43],\n",
    "#   [\"serf\", \"slave\", 0.39]\n",
    "# ]\n",
    "# for word_pair in word_pairs:\n",
    "#     print(\"%s\\t%s\\t%.2f\\t%.2f\" % (word_pair[0], word_pair[1], word_pair[2],\n",
    "#                                   word_similarity(word_pair[0], word_pair[1])))\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "sentence_pairs = [\n",
    "    [\"I like that bachelor.\", \"I like that unmarried man.\", 0.561],\n",
    "    [\"John is very nice.\", \"Is John very nice?\", 0.977],\n",
    "    [\"Red alcoholic drink.\", \"A bottle of wine.\", 0.585],\n",
    "    [\"Red alcoholic drink.\", \"Fresh orange juice.\", 0.611],\n",
    "    [\"Red alcoholic drink.\", \"An English dictionary.\", 0.0],\n",
    "    [\"Red alcoholic drink.\", \"Fresh apple juice.\", 0.420],\n",
    "    [\"A glass of cider.\", \"A full cup of apple juice.\", 0.678],\n",
    "    [\"It is a dog.\", \"That must be your dog.\", 0.739],\n",
    "    [\"It is a dog.\", \"It is a log.\", 0.623],\n",
    "    [\"It is a dog.\", \"It is a pig.\", 0.790],\n",
    "    [\"Dogs are animals.\", \"They are common pets.\", 0.738],\n",
    "    [\"Canis familiaris are animals.\", \"Dogs are common pets.\", 0.362],\n",
    "    [\"I have a pen.\", \"Where do you live?\", 0.0],\n",
    "    [\"I have a pen.\", \"Where is ink?\", 0.129],\n",
    "    [\"I have a hammer.\", \"Take some nails.\", 0.508],\n",
    "    [\"I have a hammer.\", \"Take some apples.\", 0.121]\n",
    "]\n",
    "for sent_pair in sentence_pairs:\n",
    "    print(\"%s\\t%s\\t%.3f\\t%.3f\\t%.3f\" % (sent_pair[0], sent_pair[1], sent_pair[2], \n",
    "        similarity(sent_pair[0], sent_pair[1], False),\n",
    "        similarity(sent_pair[0], sent_pair[1], True)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "def clean(path, text0, stopwords):\n",
    "    ##returns a list of lists, with words in it.\n",
    "    with open(path+text0) as txt:\n",
    "        data = txt.read()\n",
    "\n",
    "        # data = re.sub('\\\\w+\\\\set\\\\sal', '<UNK>', data)\n",
    "        data = re.sub('\\\\w+\\\\sand\\\\s+\\\\w+\\s\\\\(\\\\d{4}\\\\)', '<UNK>', data)\n",
    "        data = re.sub('[A-Z][a-z]*\\s+\\\\(\\\\d{4}\\\\)', '<UNK>', data)\n",
    "        data = re.sub('\\\\([^)]*\\\\)', '<UNK>', data)\n",
    "        data = re.sub('[A-Z][a-z]*\\\\set\\\\sal.', '<UNK>', data)\n",
    "        data = re.sub('\\s+[.]', '.', data)\n",
    "        data = re.sub('\\s+[,]', ',', data)\n",
    "        data = data.replace(u'\\xa0', u' ')\n",
    "        data = data.split('\\n')\n",
    "        data = [n.strip().split('.') for n in data]\n",
    "        data = [[z.strip().split(' ') for z in n] for n in data]\n",
    "        data = [[z for z in n if len(z) > 5] for n in data if len(n) > 0]\n",
    "        data = [n for n in data if len(n) > 0]\n",
    "        data = [[n for sublist in l for n in sublist] for l in data]\n",
    "        data = [[n.lower() for n in l if not re.search('\\\\d', n)] for l in data]\n",
    "        data = [[n for n in l if re.search('[a-zA-Z]', n)] for l in data] ####TODO check\n",
    "        data = [[n for n in l if n not in stopwords] for l in data]\n",
    "        data = [[n for n in l if '<unk>' not in n] for l in data]\n",
    "        data = [[n for n in l if len(n)>1] for l in data]\n",
    "        data = [[n.replace(',','') for n in l] for l in data]\n",
    "        data = [[n.replace(\"'s\",'') for n in l] for l in data]\n",
    "        # data = [[n.split('-') for n in l] for l in data]\n",
    "        # data = [[n for n in l if not re.search('\\\\', n)] for l in data]\n",
    "        txt.close()\n",
    "    return data\n",
    "\n",
    "papers = [n for n in os.listdir(os.path.join(root, 'papers')) if n.endswith('txt')]\n",
    "test = random.choice(papers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'C:\\\\Users\\\\ifue3702\\\\Documents\\\\GitHub\\\\Litho\\\\papers\\\\10.1016-j.carres.2004.12.026.txt'"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.path.join(os.path.join(root, 'papers'),test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "texto = open(os.path.join(os.path.join(root, 'papers'),test))\n",
    "texto.read()\n",
    "#     data = txt.read()\n",
    "#     print(data)\n",
    "texto.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [Root]",
   "language": "python",
   "name": "Python [Root]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}