simple bayes.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18846\n",
      "20\n",
      "['alt.atheism', 'comp.graphics', 'comp.os.ms-windows.misc', 'comp.sys.ibm.pc.hardware', 'comp.sys.mac.hardware', 'comp.windows.x', 'misc.forsale', 'rec.autos', 'rec.motorcycles', 'rec.sport.baseball', 'rec.sport.hockey', 'sci.crypt', 'sci.electronics', 'sci.med', 'sci.space', 'soc.religion.christian', 'talk.politics.guns', 'talk.politics.mideast', 'talk.politics.misc', 'talk.religion.misc']\n",
      "[rec.sport.hockey]:\t\t \"From: Mamatha Devineni Ratnam <mr47+@andrew.cmu.edu> ...\"\n",
      "[comp.sys.ibm.pc.hardware]:\t\t \"From: mblawson@midway.ecn.uoknor.edu (Matthew B Lawson) ...\"\n",
      "[talk.politics.mideast]:\t\t \"From: hilmi-er@dsv.su.se (Hilmi Eren) ...\"\n",
      "[comp.sys.ibm.pc.hardware]:\t\t \"From: guyd@austin.ibm.com (Guy Dawson) ...\"\n",
      "[comp.sys.mac.hardware]:\t\t \"From: Alexander Samuel McDiarmid <am2o+@andrew.cmu.edu> ...\"\n",
      "[sci.electronics]:\t\t \"From: tell@cs.unc.edu (Stephen Tell) ...\"\n",
      "[comp.sys.mac.hardware]:\t\t \"From: lpa8921@tamuts.tamu.edu (Louis Paul Adams) ...\"\n",
      "[rec.sport.hockey]:\t\t \"From: dchhabra@stpl.ists.ca (Deepak Chhabra) ...\"\n",
      "[rec.sport.hockey]:\t\t \"From: dchhabra@stpl.ists.ca (Deepak Chhabra) ...\"\n",
      "[talk.religion.misc]:\t\t \"From: arromdee@jyusenkyou.cs.jhu.edu (Ken Arromdee) ...\"\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "from sklearn.datasets import fetch_20newsgroups\n",
    "news = fetch_20newsgroups(subset='all')\n",
    " \n",
    "print(len(news.data))\n",
    "# 18846\n",
    " \n",
    "print(len(news.target_names))\n",
    "# 20\n",
    " \n",
    "print(news.target_names)\n",
    "# ['alt.atheism', 'comp.graphics', 'comp.os.ms-windows.misc', 'comp.sys.ibm.pc.hardware', 'comp.sys.mac.hardware', 'comp.windows.x', 'misc.forsale', 'rec.autos', 'rec.motorcycles', 'rec.sport.baseball', 'rec.sport.hockey', 'sci.crypt', 'sci.electronics', 'sci.med', 'sci.space', 'soc.religion.christian', 'talk.politics.guns', 'talk.politics.mideast', 'talk.politics.misc', 'talk.religion.misc']\n",
    " \n",
    "for text, num_label in zip(news.data[:10], news.target[:10]):\n",
    "    print('[%s]:\\t\\t \"%s ...\"' % (news.target_names[num_label], text[:100].split('\\n')[0]))\n",
    " \n",
    "# ['alt.atheism', 'comp.graphics', 'comp.os.ms-windows.misc', 'comp.sys.ibm.pc.hardware', 'comp.sys.mac.hardware', 'comp.windows.x', 'misc.forsale', 'rec.autos', 'rec.motorcycles', 'rec.sport.baseball', 'rec.sport.hockey', 'sci.crypt', 'sci.electronics', 'sci.med', 'sci.space', 'soc.religion.christian', 'talk.politics.guns', 'talk.politics.mideast', 'talk.politics.misc', 'talk.religion.misc']\n",
    "\n",
    "#[rec.sport.hockey]:\t\t \"From: Mamatha Devineni Ratnam <mr47+@andrew.cmu.edu> ...\"\n",
    "# [comp.sys.ibm.pc.hardware]:\t\t \"From: mblawson@midway.ecn.uoknor.edu (Matthew B Lawson) ...\"\n",
    "# [talk.politics.mideast]:\t\t \"From: hilmi-er@dsv.su.se (Hilmi Eren) ...\"\n",
    "# [comp.sys.ibm.pc.hardware]:\t\t \"From: guyd@austin.ibm.com (Guy Dawson) ...\"\n",
    "# [comp.sys.mac.hardware]:\t\t \"From: Alexander Samuel McDiarmid <am2o+@andrew.cmu.edu> ...\"\n",
    "# [sci.electronics]:\t\t \"From: tell@cs.unc.edu (Stephen Tell) ...\"\n",
    "# [comp.sys.mac.hardware]:\t\t \"From: lpa8921@tamuts.tamu.edu (Louis Paul Adams) ...\"\n",
    "# [rec.sport.hockey]:\t\t \"From: dchhabra@stpl.ists.ca (Deepak Chhabra) ...\"\n",
    "# [rec.sport.hockey]:\t\t \"From: dchhabra@stpl.ists.ca (Deepak Chhabra) ...\"\n",
    "# [talk.religion.misc]:\t\t \"From: arromdee@jyusenkyou.cs.jhu.edu (Ken Arromdee) ...\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_names = set(news.target_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'alt.atheism',\n",
       " 'comp.graphics',\n",
       " 'comp.os.ms-windows.misc',\n",
       " 'comp.sys.ibm.pc.hardware',\n",
       " 'comp.sys.mac.hardware',\n",
       " 'comp.windows.x',\n",
       " 'misc.forsale',\n",
       " 'rec.autos',\n",
       " 'rec.motorcycles',\n",
       " 'rec.sport.baseball',\n",
       " 'rec.sport.hockey',\n",
       " 'sci.crypt',\n",
       " 'sci.electronics',\n",
       " 'sci.med',\n",
       " 'sci.space',\n",
       " 'soc.religion.christian',\n",
       " 'talk.politics.guns',\n",
       " 'talk.politics.mideast',\n",
       " 'talk.politics.misc',\n",
       " 'talk.religion.misc'}"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 19.6 ms, sys: 11.4 ms, total: 31 ms\n",
      "Wall time: 48 ms\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "from sklearn.cross_validation import train_test_split\n",
    "\n",
    "def train(classifier, X, y):\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=33)\n",
    "\n",
    "    classifier.fit(X_train, y_train)\n",
    "    print(\"Accuracy: %s\" % classifier.score(X_test, y_test))\n",
    "    return classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "from nltk.stem import PorterStemmer\n",
    "from nltk import word_tokenize\n",
    "\n",
    "stemmer = PorterStemmer()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.stem import SnowballStemmer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "arabic danish dutch english finnish french german hungarian italian norwegian porter portuguese romanian russian spanish swedish\n"
     ]
    }
   ],
   "source": [
    "print(\" \".join(SnowballStemmer.languages))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "import textacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help on function preprocess_text in module textacy.preprocess:\n",
      "\n",
      "preprocess_text(text, fix_unicode=False, lowercase=False, transliterate=False, no_urls=False, no_emails=False, no_phone_numbers=False, no_numbers=False, no_currency_symbols=False, no_punct=False, no_contractions=False, no_accents=False)\n",
      "    Normalize various aspects of a raw text doc before parsing it with Spacy.\n",
      "    A convenience function for applying all other preprocessing functions in one go.\n",
      "    \n",
      "    Args:\n",
      "        text (str): raw text to preprocess\n",
      "        fix_unicode (bool): if True, fix \"broken\" unicode such as\n",
      "            mojibake and garbled HTML entities\n",
      "        lowercase (bool): if True, all text is lower-cased\n",
      "        transliterate (bool): if True, convert non-ascii characters\n",
      "            into their closest ascii equivalents\n",
      "        no_urls (bool): if True, replace all URL strings with '*URL*'\n",
      "        no_emails (bool): if True, replace all email strings with '*EMAIL*'\n",
      "        no_phone_numbers (bool): if True, replace all phone number strings\n",
      "            with '*PHONE*'\n",
      "        no_numbers (bool): if True, replace all number-like strings\n",
      "            with '*NUMBER*'\n",
      "        no_currency_symbols (bool): if True, replace all currency symbols\n",
      "            with their standard 3-letter abbreviations\n",
      "        no_punct (bool): if True, remove all punctuation (replace with\n",
      "            empty string)\n",
      "        no_contractions (bool): if True, replace *English* contractions\n",
      "            with their unshortened forms\n",
      "        no_accents (bool): if True, replace all accented characters\n",
      "            with unaccented versions; NB: if `transliterate` is True, this option\n",
      "            is redundant\n",
      "    \n",
      "    Returns:\n",
      "        str: input ``text`` processed according to function args\n",
      "    \n",
      "    Warning:\n",
      "        These changes may negatively affect subsequent NLP analysis performed\n",
      "        on the text, so choose carefully, and preprocess at your own risk!\n",
      "\n"
     ]
    }
   ],
   "source": [
    "help(textacy.preprocess_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('>', 114),\n",
       " ('the', 105),\n",
       " ('i', 48),\n",
       " ('to', 48),\n",
       " ('of', 44),\n",
       " ('and', 41),\n",
       " ('a', 38),\n",
       " ('|', 33),\n",
       " ('is', 32),\n",
       " ('email', 31)]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_words = Counter()\n",
    "for n in news.data[:10]:\n",
    "    text = textacy.preprocess_text(n, no_emails=True, no_punct=True, no_urls=True, lowercase=True)\n",
    "    words = [stemmer.stem(word) for word in word_tokenize(text)]\n",
    "    for word in words:\n",
    "        all_words[word] += 1\n",
    "all_words.most_common(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "cleaned_up_lines = []\n",
    "for line in news.data:\n",
    "    text = textacy.preprocess_text(line, no_emails=True, no_punct=True, no_urls=True, lowercase=True)\n",
    "    words = [stemmer.stem(word) for word in word_tokenize(text)]\n",
    "    cleaned_up_lines.append(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 16.8 s, sys: 432 ms, total: 17.3 s\n",
      "Wall time: 19.9 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "from gensim.models import Phrases\n",
    "\n",
    "bigram_model = Phrases(cleaned_up_lines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "frozenset()"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bigram_model.common_terms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/gensim/models/phrases.py:486: UserWarning: For a faster implementation, use the gensim.models.phrases.Phraser class\n",
      "  warnings.warn(\"For a faster implementation, use the gensim.models.phrases.Phraser class\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 37.2 s, sys: 585 ms, total: 37.8 s\n",
      "Wall time: 41.2 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "bigram_up_lines = [bigram_model[line] for line in cleaned_up_lines]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "bigram_strings = [\" \".join(line) for line in bigram_up_lines]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.8463497453310697\n",
      "CPU times: user 7.6 s, sys: 264 ms, total: 7.87 s\n",
      "Wall time: 8.23 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    " \n",
    "trial1 = Pipeline([\n",
    "    ('vectorizer', TfidfVectorizer()),\n",
    "    ('classifier', MultinomialNB()),\n",
    "])\n",
    " \n",
    "train(trial1, news.data, news.target)\n",
    "# Accuracy: 0.8463497453310697\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.8255517826825127\n",
      "CPU times: user 6.4 s, sys: 213 ms, total: 6.62 s\n",
      "Wall time: 7 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "trial_bigrams = Pipeline([\n",
    "    ('vectorizer', TfidfVectorizer()),\n",
    "    ('classifier', MultinomialNB()),\n",
    "])\n",
    " \n",
    "train(trial_bigrams, bigram_strings, news.target)\n",
    "# Accuracy: 0.8255517826825127"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.8777589134125636\n",
      "CPU times: user 8.44 s, sys: 306 ms, total: 8.75 s\n",
      "Wall time: 10.2 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "from nltk.corpus import stopwords\n",
    " \n",
    "trial2 = Pipeline([\n",
    "    ('vectorizer', TfidfVectorizer(stop_words=stopwords.words('english'))),\n",
    "    ('classifier', MultinomialNB()),\n",
    "])\n",
    " \n",
    "train(trial2, news.data, news.target)\n",
    "# Accuracy: 0.8777589134125636"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.8637521222410866\n",
      "CPU times: user 7.18 s, sys: 273 ms, total: 7.45 s\n",
      "Wall time: 8.75 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "trial_bigrams2 = Pipeline([\n",
    "    ('vectorizer', TfidfVectorizer(stop_words=stopwords.words('english'))),\n",
    "    ('classifier', MultinomialNB()),\n",
    "])\n",
    " \n",
    "train(trial_bigrams2, bigram_strings, news.target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9151103565365025\n",
      "CPU times: user 6.29 s, sys: 228 ms, total: 6.52 s\n",
      "Wall time: 6.91 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "trial_bigrams3 = Pipeline([\n",
    "    ('vectorizer', TfidfVectorizer(stop_words=stopwords.words('english'))),\n",
    "    ('classifier', MultinomialNB(alpha=0.05)),\n",
    "])\n",
    " \n",
    "train(trial_bigrams3, bigram_strings, news.target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9244482173174873\n",
      "CPU times: user 6.21 s, sys: 212 ms, total: 6.42 s\n",
      "Wall time: 6.75 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "trial_bigrams4 = Pipeline([\n",
    "    ('vectorizer', TfidfVectorizer(stop_words=stopwords.words('english'))),\n",
    "    ('classifier', MultinomialNB(alpha=0.01)),\n",
    "])\n",
    " \n",
    "train(trial_bigrams4, bigram_strings, news.target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([10])"
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trial1.predict([news.data[0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"From: Mamatha Devineni Ratnam <mr47+@andrew.cmu.edu>\\nSubject: Pens fans reactions\\nOrganization: Post Office, Carnegie Mellon, Pittsburgh, PA\\nLines: 12\\nNNTP-Posting-Host: po4.andrew.cmu.edu\\n\\n\\n\\nI am sure some bashers of Pens fans are pretty confused about the lack\\nof any kind of posts about the recent Pens massacre of the Devils. Actually,\\nI am  bit puzzled too and a bit relieved. However, I am going to put an end\\nto non-PIttsburghers' relief with a bit of praise for the Pens. Man, they\\nare killing those Devils worse than I thought. Jagr just showed you why\\nhe is much better than his regular season stats. He is also a lot\\nfo fun to watch in the playoffs. Bowman should let JAgr have a lot of\\nfun in the next couple of games since the Pens are going to beat the pulp out of Jersey anyway. I was very disappointed not to see the Islanders lose the final\\nregular season game.          PENS RULE!!!\\n\\n\""
      ]
     },
     "execution_count": 161,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "news.data[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [],
   "source": [
    "article = \"\"\"Ryan Donato scored two goals, Troy Terry had three assists and the United States beat Slovakia 5-1 in the qualification round Tuesday to advance to face the Czech Republic in the Olympic quarterfinals.\n",
    "\n",
    "\n",
    "The Recap's Winter Olympics special: sign up for our email\n",
    " Read more\n",
    "College kids again led the way for the US, which scored more against Slovakia then it did in all three preliminary-round games. James Wisniewski, Mark Arcobello and Garrett Roe also scored for the Americans, who took advantage of a 5-on-3 power play for hits on Donato and goaltender Ryan Zapolski.\n",
    "\n",
    "“We’ve had a lot of chances that we didn’t score on,” Donato said. “Just having the chemistry build and continue to build, it feels really good.”\n",
    "\n",
    "Shaking off a collision with Ladislav Nagy, Ryan Zapolski had arguably his best game of the tournament, stopping 21 of the 22 shots he faced. Zapolski and the US also beat Slovakia 2-1 in the preliminary round when Donato scored twice. With his second two-goal game, Donato equaled his father, Ted, who scored four goals for the US at the 1992 Games in Albertville.\n",
    "\n",
    "Between periods, Donato saw video of his dad in the stands and said, “I’ve never seen him smile like that before.”\n",
    "\n",
    "Slovakia goaltender Jan Laco allowed five goals on 33 shots and Peter Ceresnak scored a power-play goal for Slovakia, which became the first team eliminated from the men’s side.\n",
    "\n",
    "In other qualification games later Tuesday, Finland played South Korea, with the winner advancing to take on Canada; Slovenia faced Norway, with the winner advancing to play the “Olympic Athletes from Russia”); and Germany played Switzerland, with the winner advancing to face Sweden.\n",
    "\n",
    "After a listless first period with no goals and few scoring chances, the US wasted little time getting on the board early in the second. Terry, as he has done all Olympics, used his speed to get to the net, and Donato picked up the loose puck and beat Laco 1:36 into the period.\n",
    "\n",
    "“I thought I’ve had a good tournament the whole time,” Terry said. “It hasn’t resulted in a lot of points and that type of stuff. So I was just trying to keep my confidence and know that I just have to play the same way.”\n",
    "\n",
    "The Americans got not one but two scares 26 seconds later when Nagy ran over Zapolski and Slovakia defenseman Michal Cajovsky put a shoulder into Donato’s head in the neutral zone. Trainers attended to Donato and Zapolski as backup goaltender Brandon Maxwell stretched and prepared to go in.\n",
    "\n",
    "Donato got looked at on the bench and Zapolski took a few minutes before deciding not to leave the net. Zapolski said he figured he had a pinched nerve in his neck and couldn’t feel his hands and feet for a few minutes, while Donato had his own worries.\n",
    "\n",
    "“I had a little bit of a bloody nose,” Donato said. “It kind of felt like I might’ve broke my nose right away but it feels fine and nothing was wrong.”\n",
    "\n",
    "With Cajovsky given a match penalty – a five-minute major and an ejection – and Nagy in the penalty box for goaltender interference, the US scored 18 seconds into its 5-on-3 power play with Donato screening Laco for Wisniewski’s first goal to make it 2-0 at the 2:20 mark. Terry took advantage of all the time in the world behind the net and found an open Arcobello for a one-timer to put the U.S. up 3-0 at 13:30.\n",
    "\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([10])"
      ]
     },
     "execution_count": 164,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trial1.predict([article])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [],
   "source": [
    "ambig_art = \"\"\"\n",
    "A program is trying to attract more women into coaching to match the strong recent growth in women's sport.\n",
    "\n",
    "Football Federation SA (FFSA) has just held its first female-only Asian Football Confederation C-licence coaching course.\n",
    "\n",
    "\"There's a lot of interest in the women's game from the excellent job the Matildas have done, and certainly at the local level as well, and it cries out as the game's growing for the female coaching base to grow as well,\" FFSA's John Mundy said.\n",
    "\n",
    "The C-licence program teaches basic skills and techniques so the coaches are able to work primarily with young players.\n",
    "\n",
    "\"It's part of a two-year project where our focus and priority is female coaching,\" Mr Mundy said.\n",
    "\n",
    "\"We're looking for some of these girls learning to coach to also be instructors on courses down the track.\"\n",
    "\n",
    " A group photo of soccer players.\n",
    "PHOTO: Cristiano dos Santos Rodrigues (L) with some of the program participants and other soccer players. (ABC News: Loukas Founten)\n",
    "Mr Mundy pushed for the course after noticing there were few female coaches in the local Women's National Premier League and no female head coaches for SA's state teams.\n",
    "\n",
    "\"We think we could get to a stage where we even see females coaching the male teams,\" he said.\n",
    "\n",
    "\"A good coach is a good coach so there's no reason why a female couldn't be coaching a male team, in my view.\"\n",
    "\n",
    "The first intake of 10 women included some school teachers interested in coaching.\n",
    "\n",
    "'Male-dominated industry needs balance'\n",
    "Program director Cristiano dos Santos Rodrigues, a former Adelaide United striker, said women benefited from the all-female coaching group.\n",
    "\n",
    "\"They talk more about football, they express their knowledge of football a bit more than if they are mixed with men,\" he said.\n",
    "\n",
    "\"When they are with men they feel a bit intimidated.\"\n",
    "\n",
    "Lauren Daniel, a soccer player for 12 years, got involved in the program hoping it might provide a pathway to coaching at national or international level.\n",
    "\n",
    "\"It's quite a male-dominated industry at the moment. I just think it was a lot more relaxed having just females there and some friends as well,\" the Adelaide teacher said.\n",
    "\n",
    "\"[It's] good to get more females involved so that young people can look up and realise women can be a coach and can be really good coaches as well.\"\n",
    "\n",
    "The Matildas have been coached by a woman just once in their 40-year history, when Hesterine de Reus from the Netherlands spent 15 months in the job.\n",
    "\n",
    "There are two female head coaches among the nine W-League teams — Heather Garriock at Canberra United and Brisbane's Mel Andreatta, this year's W-League premiership coach.\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([10])"
      ]
     },
     "execution_count": 166,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trial1.predict([ambig_art])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "basb = \"\"\"\n",
    "STARKVILLE — Andy Cannizaro is expected to be out as Mississippi State’s baseball coach after just three games into his second season, according to multiple reports.\n",
    "\n",
    "The school hasn’t confirmed the dismissal, but it is expected to happen on Tuesday morning. 247Sports.com first reported the story, and WCBI has confirmed it.\n",
    "\n",
    "Cannizaro, 39, was introduced on Nov. 5, 2016, the same day John Cohen officially took over as MSU’s athletic director. Cannizaro led the Bulldogs to a super regional last season. Before joining MSU, Cannizaro was LSU’s hitting coach and recruiting director for two years.\n",
    "\n",
    "More: Andy Cannizaro wants Mississippi State to become road warriors\n",
    "\n",
    "More: Cannizaro experiences successful first year at Mississippi State\n",
    "\n",
    "More: Andy Cannizaro wants MSU to be an extension of himself\n",
    "\n",
    "He leaves with a 40-30 record after Mississippi State opened the 2018 season by being swept at Southern Miss. 247 reported Cannizaro's dismissal will be with cause, indicating it is not related to wins or losses.\n",
    "\n",
    "As of earlier tonight, players had not been notified of the move, sources said. Pitching coach Gary Henderson is expected to take over for the rest of the season, 247Sports reported. Henderson would be a logical choice; he was Kentucky's head coach for eight seasons as Cohen’s successor before joining MSU.\n",
    "\n",
    "The Bulldogs visit Jackson State on Wednesday for their fourth game of the season.  \n",
    "\n",
    "This story will be updated.\n",
    "\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [],
   "source": [
    "wind = \"\"\"Microsoft launched ARM-powered Windows 10 PCs with “all-day” battery life back in December. While HP, Asus, and Lenovo’s devices aren’t on sale just yet, we’re still waiting to hear more about the limitations of Windows 10 running on these new PCs. Microsoft published a full list of limitations last week, spotted first by Thurrott, that details what to expect from Windows 10 on ARM. This list must have been published by accident, as the software giant removed it over the weekend so only cached copies of the information are available.\n",
    "\n",
    "Only ARM64 drivers are supported. Windows 10 on ARM can run x86 apps, but it can’t use x86 drivers. That shouldn’t be a problem for most hardware, but if you have some older peripherals then it’s likely that driver support won’t be available. Windows 10 on ARM driver support will be more limited, and similar to what Windows 10 S provides.\n",
    "x64 apps are not supported. This is something we’ve known, but Windows 10 on ARM does not support emulation of x64 apps. Microsoft is planning to support these in the future at some point, though.\n",
    "Certain games and apps don’t work. Microsoft says that games and apps that use a version of OpenGL later than 1.1 or that require hardware-accelerated OpenGL won’t work on Windows 10 on ARM. Games that use anticheat technologies also won’t run on Windows 10 on ARM.\n",
    "Apps that customize the Windows experience may not work correctly. Apps like assistitive technologies or input method editors won’t work properly on Windows 10 on ARM. Also, apps that include shell extensions (icons and right-click menus in File Explorer) like Dropbox may fail. These apps will need to be compiled natively for ARM.\n",
    "Apps that assume that all ARM-based devices are running a mobile version of Windows may not work correctly. Some apps that have been coded for Windows Phone won’t work correctly and could appear in the wrong orientation or have UI layout problems. This won’t be a huge amount of apps, though.\n",
    "The Windows Hypervisor Platform is not supported on ARM. You won’t be able to run virtual machines using Hyper-V with Windows 10 on ARM.\n",
    "It seems that for most Windows users, Windows 10 on ARM will support common apps and scenarios. Microsoft’s emulation work allows you to download most 32-bit exe files from the web and install them on ARM-powered laptops. There are clearly some limitations, outlined above, but the majority of apps should run. We’re still waiting to test an ARM-powered Windows 10 laptop to see if the battery life is what has been promised, and whether performance for desktop apps is reasonable enough.\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2])"
      ]
     },
     "execution_count": 176,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trial1.predict([wind])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'comp.os.ms-windows.misc'"
      ]
     },
     "execution_count": 177,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "news.target_names[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10"
      ]
     },
     "execution_count": 158,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "news.target[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'rec.sport.hockey'"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "news.target_names[10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 17s, sys: 1.1 s, total: 1min 18s\n",
      "Wall time: 1min 24s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "partly_cleaned = [word_tokenize(line) for line in news.data]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 19.3 s, sys: 235 ms, total: 19.5 s\n",
      "Wall time: 20 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "simple_bigram_model = Phrases(partly_cleaned)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/gensim/models/phrases.py:486: UserWarning: For a faster implementation, use the gensim.models.phrases.Phraser class\n",
      "  warnings.warn(\"For a faster implementation, use the gensim.models.phrases.Phraser class\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 46 s, sys: 677 ms, total: 46.7 s\n",
      "Wall time: 48.2 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "simple_bigrams = [simple_bigram_model[line] for line in partly_cleaned]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 17.8 s, sys: 254 ms, total: 18.1 s\n",
      "Wall time: 18.7 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "simple_trigram_model = Phrases(simple_bigrams)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/gensim/models/phrases.py:486: UserWarning: For a faster implementation, use the gensim.models.phrases.Phraser class\n",
      "  warnings.warn(\"For a faster implementation, use the gensim.models.phrases.Phraser class\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 41.8 s, sys: 498 ms, total: 42.3 s\n",
      "Wall time: 43.6 s\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "\n",
    "simple_trigrams = [simple_trigram_model[line] for line in simple_bigrams]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "simple_trigrams_strings = [\" \".join(line) for line in simple_trigrams]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "306814\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[(')_Subject_:', 12266),\n",
       " (')_writes_:', 7493),\n",
       " ('(_``_`', 5579),\n",
       " ('In_article_<', 4186),\n",
       " (\"I_do_n't\", 3783),\n",
       " (':_Re_:', 2852),\n",
       " ('Organization_:_University', 2185),\n",
       " ('Subject_:_Re_:', 1659),\n",
       " ('#_#_#_#', 1018),\n",
       " ('a_lot_of', 967),\n",
       " (\"I_ca_n't\", 878),\n",
       " (')_wrote_:', 869),\n",
       " (\"I_'m_not\", 865),\n",
       " (')_R_<', 842),\n",
       " ('..._..._..._...', 669),\n",
       " ('as_well_as', 647),\n",
       " (\"''_`_@\", 636),\n",
       " ('I_would_like', 575),\n",
       " ('Inc._Lines_:', 572),\n",
       " ('world_NNTP-Posting-Host_:', 537),\n",
       " ('a_couple_of', 491),\n",
       " ('Thanks_in_advance', 478),\n",
       " ('be_able_to', 466),\n",
       " ('X-Newsreader_:_TIN_[', 457),\n",
       " ('R_<_G', 444),\n",
       " (\"'s_message_of\", 434),\n",
       " ('in_order_to', 429),\n",
       " (\"I_'d_like\", 388),\n",
       " ('Distribution_:_usa_Lines', 388),\n",
       " (\"I_'m_sure\", 377),\n",
       " ('Distribution_:_world_Organization', 362),\n",
       " (\"I_'ve_seen\", 359),\n",
       " ('USA_Lines_:', 330),\n",
       " (\"I_'ve_been\", 318),\n",
       " ('0_1_1', 305),\n",
       " ('CA_Lines_:', 301),\n",
       " (\"I_'m_not_sure\", 294),\n",
       " ('?_MR._STEPHANOPOULOS_:', 294),\n",
       " ('University_Lines_:', 285),\n",
       " ('going_to_be', 284),\n",
       " ('Distribution_:_na_Lines', 278),\n",
       " ('would_have_been', 275),\n",
       " ('Does_anyone_know', 274),\n",
       " ('Distribution_:_usa_Organization', 270),\n",
       " ('I_am_looking_for', 269),\n",
       " ('%_A86_%_A86', 267),\n",
       " ('<_7_%_Q', 265),\n",
       " ('Distribution_:_world', 259),\n",
       " (\")_''_0D\", 256),\n",
       " ('Distribution_:_world_NNTP-Posting-Host', 254)]"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import re\n",
    "from collections import Counter\n",
    "\n",
    "p = re.compile('.*_.*_.*')\n",
    "\n",
    "selected = []\n",
    "trigrams = Counter()\n",
    "\n",
    "for trigram_sentence_ext in simple_trigrams:\n",
    "    for word in trigram_sentence_ext:\n",
    "        if p.search(word):\n",
    "            selected.append(trigram_sentence_ext)\n",
    "            trigrams[word] += 1\n",
    "            \n",
    "        \n",
    "print(len(selected))\n",
    "trigrams.most_common(50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9112903225806451\n",
      "CPU times: user 7.12 s, sys: 249 ms, total: 7.37 s\n",
      "Wall time: 7.54 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "trial_tri = Pipeline([\n",
    "    ('vectorizer', TfidfVectorizer()),\n",
    "    ('classifier', MultinomialNB(alpha=0.05)),\n",
    "])\n",
    " \n",
    "train(trial_tri, simple_trigrams_strings, news.target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9140492359932089\n",
      "CPU times: user 7.27 s, sys: 259 ms, total: 7.53 s\n",
      "Wall time: 7.64 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "trial_tri1 = Pipeline([\n",
    "    ('vectorizer', TfidfVectorizer(stop_words=stopwords.words('english'))),\n",
    "    ('classifier', MultinomialNB(alpha=0.05)),\n",
    "])\n",
    " \n",
    "train(trial_tri1, simple_trigrams_strings, news.target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "simple_bigrams_strings = [\" \".join(line) for line in simple_bigrams]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9153225806451613\n",
      "CPU times: user 6.82 s, sys: 193 ms, total: 7.01 s\n",
      "Wall time: 7.08 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "trial_bi1 = Pipeline([\n",
    "    ('vectorizer', TfidfVectorizer(stop_words=stopwords.words('english'))),\n",
    "    ('classifier', MultinomialNB(alpha=0.01)),\n",
    "])\n",
    " \n",
    "train(trial_bi1, simple_bigrams_strings, news.target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "trigram_model = Phrases(bigram_up_lines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/gensim/models/phrases.py:486: UserWarning: For a faster implementation, use the gensim.models.phrases.Phraser class\n",
      "  warnings.warn(\"For a faster implementation, use the gensim.models.phrases.Phraser class\")\n"
     ]
    }
   ],
   "source": [
    "trigram_lines = [trigram_model[line] for line in bigram_up_lines]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "trigram_strings = [\" \".join(line) for line in trigram_lines]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "144649\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import re\n",
    "from collections import Counter\n",
    "\n",
    "p = re.compile('.*_.*_.*')\n",
    "\n",
    "selected = []\n",
    "trigrams = Counter()\n",
    "\n",
    "for trigram_sentence_ext in trigram_lines:\n",
    "    for word in trigram_sentence_ext:\n",
    "        if p.search(word):\n",
    "            selected.append(trigram_sentence_ext)\n",
    "            trigrams[word] += 1\n",
    "            \n",
    "        \n",
    "print(len(selected))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('ryan_cousinetc|1982_yamaha_vision', 8),\n",
       " ('xz550_black_pig', 8),\n",
       " ('dod_0863_|id_be', 8),\n",
       " ('vancouv_bc_canada_|live', 8),\n",
       " ('run_win_31', 8),\n",
       " ('distribut_worldloc_organ_univers', 8),\n",
       " ('from_emailedu_david', 8),\n",
       " ('isa_bu_speed', 8),\n",
       " ('have_bond_bat', 8),\n",
       " ('from_email_rob_dobson', 8),\n",
       " ('ny_distribut_na', 8),\n",
       " ('organ_informix_softwar_inc', 8),\n",
       " ('from_email_kevin_marshal', 8),\n",
       " ('should_liabil_insur', 8),\n",
       " ('april_22_1993', 8),\n",
       " ('from_email_dave_laudicina', 8),\n",
       " ('nntppostinghost_diladpuncedu_organ_unc', 8),\n",
       " ('email_hamaza_h_salah', 8),\n",
       " ('those_massiv_concret', 8),\n",
       " ('at_nuclear_poer', 8),\n",
       " ('look_like_cylind', 8),\n",
       " ('have_been_pinch', 8),\n",
       " ('middl_doe_anybodi', 8),\n",
       " ('from_email_michael_beavington', 8),\n",
       " ('radiu_exactli_fit', 8),\n",
       " ('pl8_xpostedfrom_veneziarockefelleredu_nntppostinghost', 8),\n",
       " ('veri_much_in_advanc', 8),\n",
       " ('e_alan_idler', 8),\n",
       " ('from_email_peter_walker', 8),\n",
       " ('rodger_c_scoggin', 8),\n",
       " ('from_email_charl_kinci', 8),\n",
       " ('lincoln_lab_group', 8),\n",
       " ('keyword_galileo_jpl', 8),\n",
       " ('from_email_miller_jimmi', 8),\n",
       " ('subject_re_batffbi_reveng', 8),\n",
       " ('houston_administr_comput', 8),\n",
       " ('semper_fi_jammer_jim', 8),\n",
       " ('miller_texa_am', 8),\n",
       " ('ordinari_man_rik_emmet', 8),\n",
       " ('gil_moor_mike_levin', 8),\n",
       " ('memori_univers_stjohn', 8),\n",
       " ('atari_2600_processor', 8),\n",
       " ('held_respons_for', 8),\n",
       " ('newcastl_upon_tyne_uk', 8),\n",
       " ('physic_michigan_tech_houghton', 8),\n",
       " ('pgp_public_key', 8),\n",
       " ('watt_per_channel', 8),\n",
       " ('veri_good_review', 8),\n",
       " ('softwar_metric_inc_waterloo', 8),\n",
       " ('your_mouth_now', 8)]"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trigrams.most_common()[5000:5050]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9227504244482173\n",
      "CPU times: user 5.9 s, sys: 198 ms, total: 6.1 s\n",
      "Wall time: 6.22 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "trial_tri1 = Pipeline([\n",
    "    ('vectorizer', TfidfVectorizer(stop_words=stopwords.words('english'))),\n",
    "    ('classifier', MultinomialNB(alpha=0.01)),\n",
    "])\n",
    " \n",
    "train(trial_tri1, trigram_strings, news.target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9102292020373515\n",
      "CPU times: user 8.02 s, sys: 277 ms, total: 8.3 s\n",
      "Wall time: 9.09 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "trial3 = Pipeline([\n",
    "    ('vectorizer', TfidfVectorizer(stop_words=stopwords.words('english'))),\n",
    "    ('classifier', MultinomialNB(alpha=0.05)),\n",
    "])\n",
    " \n",
    "train(trial3, news.data, news.target)\n",
    "# Accuracy: 0.9102292020373515\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9061969439728353\n",
      "CPU times: user 8.3 s, sys: 287 ms, total: 8.59 s\n",
      "Wall time: 9.16 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "trial3_1 = Pipeline([\n",
    "    ('vectorizer', TfidfVectorizer(stop_words=[])),\n",
    "    ('classifier', MultinomialNB(alpha=0.05)),\n",
    "])\n",
    " \n",
    "train(trial3_1, news.data, news.target)\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9028013582342954\n",
      "CPU times: user 6.85 s, sys: 264 ms, total: 7.12 s\n",
      "Wall time: 7.6 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "trial4 = Pipeline([\n",
    "    ('vectorizer', TfidfVectorizer(stop_words=stopwords.words('english'),\n",
    "                             min_df=5)),\n",
    "    ('classifier', MultinomialNB(alpha=0.05)),\n",
    "])\n",
    " \n",
    "train(trial4, news.data, news.target)\n",
    "# Accuracy: 0.9028013582342954\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([9])"
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trial4.predict([basb])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9108658743633277\n",
      "CPU times: user 3min 38s, sys: 1.06 s, total: 3min 39s\n",
      "Wall time: 3min 45s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "import string\n",
    "from nltk.stem import PorterStemmer\n",
    "from nltk import word_tokenize\n",
    " \n",
    "def stemming_tokenizer(text):\n",
    "    stemmer = PorterStemmer()\n",
    "    return [stemmer.stem(w) for w in word_tokenize(text)]\n",
    " \n",
    "trial5 = Pipeline([\n",
    "    ('vectorizer', TfidfVectorizer(tokenizer=stemming_tokenizer,\n",
    "                             stop_words=stopwords.words('english') + list(string.punctuation))),\n",
    "    ('classifier', MultinomialNB(alpha=0.05)),\n",
    "])\n",
    " \n",
    "train(trial5, news.data, news.target)\n",
    "# Accuracy: 0.910653650255"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim.corpora import Dictionary, MmCorpus\n",
    "from gensim.models.ldamulticore import LdaMulticore\n",
    "\n",
    "import pyLDAvis\n",
    "import pyLDAvis.gensim\n",
    "import warnings\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim.utils import smart_open, simple_preprocess\n",
    "from gensim.corpora.wikicorpus import _extract_pages, filter_wiki\n",
    "from gensim.parsing.preprocessing import STOPWORDS\n",
    "\n",
    "def tokenize(text):\n",
    "    return [token for token in simple_preprocess(text) if token not in STOPWORDS]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import and setup modules we'll be using in this notebook\n",
    "import logging\n",
    "import itertools\n",
    "\n",
    "import numpy as np\n",
    "import gensim\n",
    "\n",
    "logging.basicConfig(format='%(levelname)s : %(message)s', level=logging.INFO)\n",
    "logging.root.level = logging.INFO  # ipython sometimes messes up the logging setup; restore\n",
    "\n",
    "def head(stream, n=10):\n",
    "    \"\"\"Convenience fnc: return the first `n` elements of the stream, as plain list.\"\"\"\n",
    "    return list(itertools.islice(stream, n))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO : adding document #0 to Dictionary(0 unique tokens: [])\n",
      "INFO : adding document #10000 to Dictionary(189119 unique tokens: ['!', \"'\", ',', '.', '12']...)\n",
      "INFO : built Dictionary(285782 unique tokens: ['!', \"'\", ',', '.', '12']...) from 18846 documents (total 7531964 corpus positions)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dictionary(285782 unique tokens: ['!', \"'\", ',', '.', '12']...)\n",
      "CPU times: user 10.1 s, sys: 3.78 s, total: 13.9 s\n",
      "Wall time: 17.1 s\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "\n",
    "id2word_wiki = gensim.corpora.Dictionary(partly_cleaned)\n",
    "print(id2word_wiki)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO : discarding 271859 tokens: [('!', 7444), (\"'\", 2860), (',', 18167), ('.', 18378), (':', 18846), ('<', 9698), ('>', 12205), ('@', 18836), ('Devineni', 12), ('From', 18846)]...\n",
      "INFO : keeping 13923 tokens which were in no less than 20 and no more than 1884 (=10.0%) documents\n",
      "INFO : resulting dictionary: Dictionary(13923 unique tokens: ['12', 'Actually', 'Bowman', 'Carnegie', 'Devils']...)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dictionary(13923 unique tokens: ['12', 'Actually', 'Bowman', 'Carnegie', 'Devils']...)\n"
     ]
    }
   ],
   "source": [
    "# ignore words that appear in less than 20 documents or more than 10% documents\n",
    "id2word_wiki.filter_extremes(no_below=20, no_above=0.1)\n",
    "print(id2word_wiki)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(667, 2), (1147, 1), (2701, 1), (5020, 2), (5046, 1)]\n"
     ]
    }
   ],
   "source": [
    "doc = \"A blood cell, also called a hematocyte, is a cell produced by hematopoiesis and normally found in blood.\"\n",
    "bow = id2word_wiki.doc2bow(tokenize(doc))\n",
    "print(bow)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "employees\n"
     ]
    }
   ],
   "source": [
    "print(id2word_wiki[10882])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'blood'"
      ]
     },
     "execution_count": 149,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "id2word_wiki[667]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus = (id2word_wiki.doc2bow(doc) for doc in partly_cleaned)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['From',\n",
       " ':',\n",
       " 'jackw',\n",
       " '@',\n",
       " 'boi.hp.com',\n",
       " '(',\n",
       " 'jack',\n",
       " 'wood',\n",
       " ')',\n",
       " 'Subject',\n",
       " ':',\n",
       " 'Re',\n",
       " ':',\n",
       " 'Chevy/GMC',\n",
       " '4x4',\n",
       " 'Fullsize',\n",
       " 'Pickups',\n",
       " ',',\n",
       " 'Opinions',\n",
       " '?',\n",
       " 'Distribution',\n",
       " ':',\n",
       " 'na',\n",
       " 'Organization',\n",
       " ':',\n",
       " 'Hewlett-Packard',\n",
       " '/',\n",
       " 'Boise',\n",
       " ',',\n",
       " 'Idaho',\n",
       " 'X-Newsreader',\n",
       " ':',\n",
       " 'TIN',\n",
       " '[',\n",
       " 'version',\n",
       " '1.1.4',\n",
       " 'PL6',\n",
       " ']',\n",
       " 'Lines',\n",
       " ':',\n",
       " '41',\n",
       " 'Dick',\n",
       " 'Grady',\n",
       " '(',\n",
       " 'grady',\n",
       " '@',\n",
       " 'world.std.com',\n",
       " ')',\n",
       " 'wrote',\n",
       " ':',\n",
       " ':',\n",
       " ':',\n",
       " 'I',\n",
       " 'am',\n",
       " 'considering',\n",
       " 'buying',\n",
       " 'a',\n",
       " '1993',\n",
       " 'Chevy',\n",
       " 'or',\n",
       " 'GMC',\n",
       " '4x4',\n",
       " 'full-size',\n",
       " 'pickup',\n",
       " 'with',\n",
       " ':',\n",
       " 'the',\n",
       " 'extended',\n",
       " 'cab',\n",
       " '.',\n",
       " 'Any',\n",
       " 'opinions',\n",
       " 'about',\n",
       " 'these',\n",
       " 'vehicles',\n",
       " '?',\n",
       " 'Have',\n",
       " 'there',\n",
       " 'been',\n",
       " ':',\n",
       " 'any',\n",
       " 'significant',\n",
       " 'problems',\n",
       " '?',\n",
       " ':',\n",
       " ':',\n",
       " '--',\n",
       " ':',\n",
       " 'Dick',\n",
       " 'Grady',\n",
       " 'Salem',\n",
       " ',',\n",
       " 'NH',\n",
       " ',',\n",
       " 'USA',\n",
       " 'grady',\n",
       " '@',\n",
       " 'world.std.com',\n",
       " ':',\n",
       " 'So',\n",
       " 'many',\n",
       " 'newsgroups',\n",
       " ',',\n",
       " 'so',\n",
       " 'little',\n",
       " 'time',\n",
       " '!',\n",
       " 'I',\n",
       " 'bought',\n",
       " 'a',\n",
       " 'brand',\n",
       " 'new',\n",
       " '1992',\n",
       " 'Chevrolet',\n",
       " 'K2500',\n",
       " 'HD',\n",
       " '4x4',\n",
       " 'extended',\n",
       " 'cab',\n",
       " 'last',\n",
       " 'May',\n",
       " '.',\n",
       " 'It',\n",
       " 'has',\n",
       " 'had',\n",
       " 'many',\n",
       " ',',\n",
       " 'many',\n",
       " 'problems',\n",
       " '.',\n",
       " 'See',\n",
       " 'my',\n",
       " 'earler',\n",
       " 'post',\n",
       " 'that',\n",
       " 'describes',\n",
       " 'the',\n",
       " 'situation',\n",
       " '.',\n",
       " 'I',\n",
       " 'went',\n",
       " 'to',\n",
       " 'BBB',\n",
       " 'arbitration',\n",
       " ',',\n",
       " 'and',\n",
       " 'they',\n",
       " 'ruled',\n",
       " 'that',\n",
       " 'Chevrolet',\n",
       " 'must',\n",
       " 'buy',\n",
       " 'it',\n",
       " 'back',\n",
       " 'from',\n",
       " 'me',\n",
       " '.',\n",
       " 'If',\n",
       " 'you',\n",
       " 'do',\n",
       " 'get',\n",
       " 'one',\n",
       " ',',\n",
       " 'stay',\n",
       " 'away',\n",
       " 'from',\n",
       " 'the',\n",
       " '5',\n",
       " 'speed',\n",
       " 'manual',\n",
       " 'with',\n",
       " 'the',\n",
       " 'deep',\n",
       " 'low',\n",
       " 'first',\n",
       " 'gear',\n",
       " '.',\n",
       " 'They',\n",
       " 'have',\n",
       " 'put',\n",
       " 'three',\n",
       " 'of',\n",
       " 'them',\n",
       " 'in',\n",
       " 'my',\n",
       " 'truck',\n",
       " 'so',\n",
       " 'far',\n",
       " '.',\n",
       " 'After',\n",
       " 'about',\n",
       " '1,500',\n",
       " 'miles',\n",
       " ',',\n",
       " 'overdrive',\n",
       " 'either',\n",
       " 'starts',\n",
       " 'rattling',\n",
       " 'or',\n",
       " 'hissing',\n",
       " 'loudly',\n",
       " '.',\n",
       " 'There',\n",
       " 'is',\n",
       " 'no',\n",
       " 'way',\n",
       " 'to',\n",
       " 'fix',\n",
       " 'them',\n",
       " '.',\n",
       " 'Chevrolet',\n",
       " 'says',\n",
       " 'that',\n",
       " 'the',\n",
       " 'noise',\n",
       " 'is',\n",
       " '``',\n",
       " 'a',\n",
       " 'characteristic',\n",
       " 'of',\n",
       " 'the',\n",
       " 'transmission',\n",
       " '.',\n",
       " \"''\",\n",
       " 'Also',\n",
       " ',',\n",
       " 'if',\n",
       " 'you',\n",
       " 'are',\n",
       " 'planning',\n",
       " 'to',\n",
       " 'use',\n",
       " 'your',\n",
       " 'truck',\n",
       " 'to',\n",
       " 'tow',\n",
       " ',',\n",
       " 'the',\n",
       " 'gear',\n",
       " 'ratios',\n",
       " 'in',\n",
       " 'that',\n",
       " 'tranny',\n",
       " 'suck',\n",
       " '.',\n",
       " 'On',\n",
       " 'a',\n",
       " 'steep',\n",
       " 'hill',\n",
       " ',',\n",
       " 'you',\n",
       " 'get',\n",
       " 'up',\n",
       " 'to',\n",
       " 'about',\n",
       " '55',\n",
       " 'MPH',\n",
       " 'in',\n",
       " 'second',\n",
       " 'gear',\n",
       " 'at',\n",
       " '4,000',\n",
       " 'RPM',\n",
       " '(',\n",
       " 'yellow',\n",
       " 'line',\n",
       " ')',\n",
       " '.',\n",
       " 'If',\n",
       " 'you',\n",
       " 'shift',\n",
       " 'to',\n",
       " 'third',\n",
       " ',',\n",
       " 'the',\n",
       " 'RPM',\n",
       " 'drop',\n",
       " 'to',\n",
       " 'only',\n",
       " '2,500',\n",
       " ',',\n",
       " 'and',\n",
       " 'you',\n",
       " 'begin',\n",
       " 'to',\n",
       " 'loose',\n",
       " 'speed',\n",
       " '.',\n",
       " 'I',\n",
       " 'should',\n",
       " 'point',\n",
       " 'out',\n",
       " 'that',\n",
       " 'the',\n",
       " '350',\n",
       " 'V8',\n",
       " 'they',\n",
       " 'put',\n",
       " 'in',\n",
       " 'the',\n",
       " 'HD',\n",
       " '(',\n",
       " '8600',\n",
       " 'GVW',\n",
       " ')',\n",
       " 'trucks',\n",
       " 'is',\n",
       " 'a',\n",
       " 'detuned',\n",
       " 'motor',\n",
       " 'compared',\n",
       " 'to',\n",
       " 'the',\n",
       " 'one',\n",
       " 'they',\n",
       " 'put',\n",
       " 'in',\n",
       " 'the',\n",
       " 'light',\n",
       " 'duty',\n",
       " 'ones',\n",
       " '.',\n",
       " 'They',\n",
       " 'dropped',\n",
       " 'the',\n",
       " 'compression',\n",
       " 'ratio',\n",
       " ',',\n",
       " 'supposedly',\n",
       " 'for',\n",
       " '``',\n",
       " 'engine',\n",
       " 'longevity',\n",
       " \"''\",\n",
       " 'reasons',\n",
       " '.',\n",
       " 'So',\n",
       " 'the',\n",
       " 'light',\n",
       " 'duty',\n",
       " '350',\n",
       " 'may',\n",
       " 'pull',\n",
       " 'better',\n",
       " 'than',\n",
       " 'my',\n",
       " 'truck',\n",
       " 'does',\n",
       " '.',\n",
       " 'Other',\n",
       " 'things',\n",
       " 'that',\n",
       " 'have',\n",
       " 'gone',\n",
       " 'wrong',\n",
       " 'include',\n",
       " 'the',\n",
       " 'ventilation',\n",
       " 'fan',\n",
       " '(',\n",
       " '3',\n",
       " 'times',\n",
       " 'so',\n",
       " 'far',\n",
       " ')',\n",
       " ',',\n",
       " 'paint',\n",
       " '(',\n",
       " 'had',\n",
       " 'specs',\n",
       " 'of',\n",
       " 'rust',\n",
       " 'embedded',\n",
       " 'in',\n",
       " 'the',\n",
       " 'paint',\n",
       " 'from',\n",
       " 'being',\n",
       " 'shipped',\n",
       " 'by',\n",
       " 'rail',\n",
       " 'with',\n",
       " 'no',\n",
       " 'covering',\n",
       " ')',\n",
       " ',',\n",
       " 'and',\n",
       " 'suspension',\n",
       " 'parts',\n",
       " '(',\n",
       " 'link',\n",
       " 'between',\n",
       " 'stabilizer',\n",
       " 'and',\n",
       " 'control',\n",
       " 'arm',\n",
       " 'fell',\n",
       " 'off',\n",
       " ')',\n",
       " '.',\n",
       " 'Any',\n",
       " 'company',\n",
       " 'can',\n",
       " 'make',\n",
       " 'a',\n",
       " 'bad',\n",
       " 'individual',\n",
       " 'car',\n",
       " ',',\n",
       " 'Chevrolet',\n",
       " 'included',\n",
       " '.',\n",
       " 'What',\n",
       " 'really',\n",
       " 'bothered',\n",
       " 'me',\n",
       " 'was',\n",
       " 'the',\n",
       " 'way',\n",
       " 'they',\n",
       " 'reacted',\n",
       " '.',\n",
       " 'They',\n",
       " 'made',\n",
       " 'no',\n",
       " 'attempt',\n",
       " 'to',\n",
       " 'deal',\n",
       " 'with',\n",
       " 'me',\n",
       " 'except',\n",
       " 'to',\n",
       " 'tell',\n",
       " 'me',\n",
       " 'to',\n",
       " 'take',\n",
       " 'it',\n",
       " 'back',\n",
       " 'to',\n",
       " 'the',\n",
       " 'dealer',\n",
       " 'for',\n",
       " 'them',\n",
       " 'to',\n",
       " 'attempt',\n",
       " 'to',\n",
       " 'fix',\n",
       " 'it',\n",
       " 'one',\n",
       " 'more',\n",
       " 'time',\n",
       " '.',\n",
       " 'So',\n",
       " 'I',\n",
       " 'bought',\n",
       " 'a',\n",
       " 'brand',\n",
       " 'new',\n",
       " 'Ford',\n",
       " 'F250',\n",
       " 'HD',\n",
       " 'Super',\n",
       " 'Cab',\n",
       " 'with',\n",
       " 'a',\n",
       " '460',\n",
       " 'and',\n",
       " 'an',\n",
       " 'automatic',\n",
       " '.',\n",
       " 'I',\n",
       " 'will',\n",
       " 'never',\n",
       " 'buy',\n",
       " 'another',\n",
       " 'Chevrolet',\n",
       " '.',\n",
       " 'jackw',\n",
       " '@',\n",
       " 'hpdmd48.boi.hp.com']"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = partly_cleaned[20]\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [],
   "source": [
    "from itertools import chain\n",
    "\n",
    "corpus_counter = Counter([word for words in partly_cleaned for word in words ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('>', 265526),\n",
       " (',', 263523),\n",
       " ('.', 237630),\n",
       " ('the', 213298),\n",
       " (':', 185877),\n",
       " ('--', 178752),\n",
       " ('to', 117987),\n",
       " (')', 113315),\n",
       " ('of', 111837),\n",
       " ('(', 110932)]"
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus_counter.most_common(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9436"
      ]
     },
     "execution_count": 143,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus_counter['so']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:90: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr))[0]\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:90: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr))[0]\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:90: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr))[0]\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:90: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr))[0]\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:90: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr))[0]\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:90: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr))[0]\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:90: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr))[0]\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:90: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr))[0]\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:90: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr))[0]\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:90: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr))[0]\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:90: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr))[0]\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:90: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr))[0]\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:90: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr))[0]\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:90: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr))[0]\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:90: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr))[0]\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:90: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr))[0]\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:90: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr))[0]\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:90: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr))[0]\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:90: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr))[0]\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:90: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr))[0]\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:90: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr))[0]\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:90: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr))[0]\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:90: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr))[0]\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:90: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr))[0]\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:90: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr))[0]\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:90: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr))[0]\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:90: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr))[0]\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:90: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr))[0]\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:90: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr))[0]\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:90: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr))[0]\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:90: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr))[0]\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:90: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr))[0]\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:90: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr))[0]\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:90: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr))[0]\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:90: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr))[0]\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:90: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr))[0]\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:90: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr))[0]\n",
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/msgpack_numpy.py:86: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  dtype=np.dtype(descr)).reshape(obj[b'shape'])\n"
     ]
    }
   ],
   "source": [
    "import spacy\n",
    "nlp = spacy.load('en')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "384"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(nlp('Chevrolet').vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# id2word_wiki.doc2bow(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# vector = next(iter(corpus))\n",
    "# print(vector) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "i 12\n"
     ]
    }
   ],
   "source": [
    "# what is the most common word in that first article?\n",
    "def maxfunc(param):\n",
    "    word_index, count = param\n",
    "    return count\n",
    "most_index, most_count = max(vector, key=maxfunc)\n",
    "print(id2word_wiki[most_index], most_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO : storing corpus in Matrix Market format to ./data/corpus_bow.mm\n",
      "INFO : saving sparse matrix to ./data/corpus_bow.mm\n",
      "INFO : PROGRESS: saving document #0\n",
      "INFO : PROGRESS: saving document #1000\n",
      "INFO : PROGRESS: saving document #2000\n",
      "INFO : PROGRESS: saving document #3000\n",
      "INFO : PROGRESS: saving document #4000\n",
      "INFO : PROGRESS: saving document #5000\n",
      "INFO : PROGRESS: saving document #6000\n",
      "INFO : PROGRESS: saving document #7000\n",
      "INFO : PROGRESS: saving document #8000\n",
      "INFO : PROGRESS: saving document #9000\n",
      "INFO : PROGRESS: saving document #10000\n",
      "INFO : PROGRESS: saving document #11000\n",
      "INFO : PROGRESS: saving document #12000\n",
      "INFO : PROGRESS: saving document #13000\n",
      "INFO : PROGRESS: saving document #14000\n",
      "INFO : PROGRESS: saving document #15000\n",
      "INFO : PROGRESS: saving document #16000\n",
      "INFO : PROGRESS: saving document #17000\n",
      "INFO : PROGRESS: saving document #18000\n",
      "INFO : saved 18846x13923 matrix, density=0.598% (1568478/262392858)\n",
      "INFO : saving MmCorpus index to ./data/corpus_bow.mm.index\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 9.6 s, sys: 277 ms, total: 9.88 s\n",
      "Wall time: 11.2 s\n"
     ]
    }
   ],
   "source": [
    "%time gensim.corpora.MmCorpus.serialize('./data/corpus_bow.mm', corpus)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO : loaded corpus index from ./data/corpus_bow.mm.index\n",
      "INFO : initializing corpus reader from ./data/corpus_bow.mm\n",
      "INFO : accepted corpus with 18846 documents, 13923 features, 1568478 non-zero entries\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MmCorpus(18846 documents, 13923 features, 1568478 non-zero entries)\n"
     ]
    }
   ],
   "source": [
    "mm_corpus = gensim.corpora.MmCorpus('./data/corpus_bow.mm')\n",
    "print(mm_corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO : using symmetric alpha at 0.05\n",
      "INFO : using symmetric eta at 0.05\n",
      "INFO : using serial LDA version on this node\n",
      "INFO : running online (multi-pass) LDA training, 20 topics, 4 passes over the supplied corpus of 4000 documents, updating model once every 2000 documents, evaluating perplexity every 4000 documents, iterating 50x with a convergence threshold of 0.001000\n",
      "WARNING : too few updates, training might not converge; consider increasing the number of passes or iterations to improve accuracy\n",
      "INFO : PROGRESS: pass 0, at document #2000/4000\n",
      "INFO : merging changes from 2000 documents into a model of 4000 documents\n",
      "INFO : topic #4 (0.050): 0.005*\"*\" + 0.004*\"/\" + 0.003*\"system\" + 0.003*\"_\" + 0.003*\"data\" + 0.002*\"enough\" + 0.002*\"8\" + 0.002*\"4\" + 0.002*\"#\" + 0.002*\"netcom.com\"\n",
      "INFO : topic #12 (0.050): 0.006*\"#\" + 0.003*\"drive\" + 0.003*\"him\" + 0.002*\"%\" + 0.002*\"system\" + 0.002*\"our\" + 0.002*\"6\" + 0.002*\"seems\" + 0.002*\"David\" + 0.002*\"4\"\n",
      "INFO : topic #11 (0.050): 0.003*\"#\" + 0.003*\"God\" + 0.003*\"JPEG\" + 0.003*\"system\" + 0.002*\"*\" + 0.002*\"4\" + 0.002*\"law\" + 0.002*\"=\" + 0.002*\"file\" + 0.002*\"both\"\n",
      "INFO : topic #19 (0.050): 0.065*\"#\" + 0.005*\"%\" + 0.003*\"/\" + 0.003*\"`\" + 0.003*\"*\" + 0.003*\"M\" + 0.002*\"4\" + 0.002*\"6\" + 0.002*\"5\" + 0.002*\"read\"\n",
      "INFO : topic #18 (0.050): 0.003*\"window\" + 0.002*\"program\" + 0.002*\"8\" + 0.002*\"system\" + 0.002*\"10\" + 0.002*\"our\" + 0.002*\"God\" + 0.002*\"please\" + 0.002*\"He\" + 0.002*\"problems\"\n",
      "INFO : topic diff=5.934305, rho=1.000000\n",
      "INFO : -9.463 per-word bound, 705.9 perplexity estimate based on a held-out corpus of 2000 documents with 261947 words\n",
      "INFO : PROGRESS: pass 0, at document #4000/4000\n",
      "INFO : merging changes from 2000 documents into a model of 4000 documents\n",
      "INFO : topic #13 (0.050): 0.004*\"system\" + 0.002*\"God\" + 0.002*\"He\" + 0.002*\"David\" + 0.002*\"opinions\" + 0.002*\"him\" + 0.002*\"John\" + 0.002*\"government\" + 0.002*\"Jesus\" + 0.002*\"New\"\n",
      "INFO : topic #16 (0.050): 0.003*\"us\" + 0.002*\"We\" + 0.002*\"our\" + 0.002*\"him\" + 0.002*\"God\" + 0.002*\"He\" + 0.002*\"must\" + 0.002*\"No\" + 0.002*\"de\" + 0.002*\"team\"\n",
      "INFO : topic #9 (0.050): 0.121*\"`\" + 0.097*\"%\" + 0.080*\"#\" + 0.038*\"M\" + 0.034*\"X\" + 0.018*\"G\" + 0.017*\"R\" + 0.016*\"0\" + 0.013*\"=\" + 0.012*\"+\"\n",
      "INFO : topic #2 (0.050): 0.008*\"}\" + 0.007*\"DOS\" + 0.007*\"*/\" + 0.007*\"X\" + 0.006*\"/*\" + 0.006*\"0\" + 0.005*\"*\" + 0.004*\"system\" + 0.004*\"{\" + 0.004*\"key\"\n",
      "INFO : topic #8 (0.050): 0.010*\"Jesus\" + 0.008*\"God\" + 0.004*\"appears\" + 0.004*\"He\" + 0.003*\"Christian\" + 0.003*\"Matthew\" + 0.003*\"Christ\" + 0.003*\"Bible\" + 0.003*\"our\" + 0.003*\"His\"\n",
      "INFO : topic diff=2.171029, rho=0.707107\n",
      "INFO : PROGRESS: pass 1, at document #2000/4000\n",
      "INFO : merging changes from 2000 documents into a model of 4000 documents\n",
      "INFO : topic #10 (0.050): 0.006*\"_/\" + 0.005*\"#\" + 0.003*\"He\" + 0.003*\"God\" + 0.003*\"car\" + 0.002*\"us\" + 0.002*\"him\" + 0.002*\"our\" + 0.002*\"down\" + 0.002*\"+\"\n",
      "INFO : topic #0 (0.050): 0.037*\"%\" + 0.006*\"Israel\" + 0.004*\"#\" + 0.004*\"Turkey\" + 0.003*\"Armenians\" + 0.003*\"`\" + 0.002*\"population\" + 0.002*\"Machines\" + 0.002*\"4\" + 0.002*\"ED\"\n",
      "INFO : topic #11 (0.050): 0.008*\"JPEG\" + 0.004*\"God\" + 0.004*\"image\" + 0.004*\"file\" + 0.003*\"GIF\" + 0.003*\"Q\" + 0.003*\"law\" + 0.002*\"us\" + 0.002*\"Israel\" + 0.002*\"system\"\n",
      "INFO : topic #4 (0.050): 0.004*\"data\" + 0.004*\"system\" + 0.004*\"*\" + 0.004*\"/\" + 0.003*\"netcom.com\" + 0.003*\"image\" + 0.002*\"bit\" + 0.002*\"software\" + 0.002*\"8\" + 0.002*\"enough\"\n",
      "INFO : topic #5 (0.050): 0.005*\"Armenian\" + 0.004*\"Turkish\" + 0.003*\"our\" + 0.003*\"We\" + 0.003*\"Armenia\" + 0.003*\"government\" + 0.002*\"THE\" + 0.002*\"Armenians\" + 0.002*\"Azerbaijan\" + 0.002*\"law\"\n",
      "INFO : topic diff=1.336163, rho=0.500000\n",
      "INFO : -8.737 per-word bound, 426.7 perplexity estimate based on a held-out corpus of 2000 documents with 261947 words\n",
      "INFO : PROGRESS: pass 1, at document #4000/4000\n",
      "INFO : merging changes from 2000 documents into a model of 4000 documents\n",
      "INFO : topic #5 (0.050): 0.005*\"Armenian\" + 0.005*\"Turkish\" + 0.003*\"Azerbaijan\" + 0.003*\"government\" + 0.003*\"Armenia\" + 0.003*\"Armenians\" + 0.003*\"our\" + 0.003*\"We\" + 0.003*\"THE\" + 0.003*\"Muslims\"\n",
      "INFO : topic #19 (0.050): 0.085*\"#\" + 0.003*\"Windows\" + 0.003*\"water\" + 0.003*\"Mac\" + 0.002*\"/\" + 0.002*\"PC\" + 0.002*\"No\" + 0.002*\"read\" + 0.002*\"children\" + 0.002*\"version\"\n",
      "INFO : topic #0 (0.050): 0.028*\"%\" + 0.008*\"Israel\" + 0.005*\"Machines\" + 0.004*\"population\" + 0.003*\"Turkey\" + 0.003*\"#\" + 0.003*\"Istanbul\" + 0.003*\"Armenians\" + 0.003*\"Israeli\" + 0.003*\"/|\"\n",
      "INFO : topic #4 (0.050): 0.005*\"data\" + 0.004*\"system\" + 0.004*\"software\" + 0.003*\"keyboard\" + 0.003*\"image\" + 0.003*\"netcom.com\" + 0.003*\"/\" + 0.003*\"*\" + 0.003*\"version\" + 0.003*\"available\"\n",
      "INFO : topic #6 (0.050): 0.019*\"/\" + 0.010*\"\\\" + 0.006*\"_\" + 0.005*\"card\" + 0.004*\"___\" + 0.003*\"Doug\" + 0.003*\"operational\" + 0.003*\"mhz\" + 0.003*\"Apr\" + 0.003*\"25\"\n",
      "INFO : topic diff=1.381630, rho=0.500000\n",
      "INFO : PROGRESS: pass 2, at document #2000/4000\n",
      "INFO : merging changes from 2000 documents into a model of 4000 documents\n",
      "INFO : topic #1 (0.050): 0.056*\"X\" + 0.006*\"speed\" + 0.005*\"drive\" + 0.005*\"window\" + 0.005*\"IDE\" + 0.005*\"file\" + 0.005*\"output\" + 0.005*\"SCSI\" + 0.005*\"server\" + 0.004*\"program\"\n",
      "INFO : topic #10 (0.050): 0.006*\"_/\" + 0.004*\"car\" + 0.003*\"He\" + 0.003*\"us\" + 0.003*\"him\" + 0.003*\"down\" + 0.003*\"#\" + 0.002*\"she\" + 0.002*\"police\" + 0.002*\"her\"\n",
      "INFO : topic #2 (0.050): 0.009*\"}\" + 0.008*\"DOS\" + 0.007*\"key\" + 0.006*\"system\" + 0.006*\"file\" + 0.005*\"{\" + 0.005*\"*\" + 0.005*\"*/\" + 0.004*\"=\" + 0.004*\"/*\"\n",
      "INFO : topic #16 (0.050): 0.003*\"us\" + 0.002*\"de\" + 0.002*\"our\" + 0.002*\"him\" + 0.002*\"again\" + 0.002*\"either\" + 0.002*\"little\" + 0.002*\"No\" + 0.002*\"must\" + 0.002*\"We\"\n",
      "INFO : topic #8 (0.050): 0.021*\"God\" + 0.016*\"Jesus\" + 0.007*\"Bible\" + 0.007*\"Christian\" + 0.007*\"Christ\" + 0.006*\"He\" + 0.006*\"Christians\" + 0.006*\"our\" + 0.005*\"human\" + 0.004*\"Christianity\"\n",
      "INFO : topic diff=1.117093, rho=0.447214\n",
      "INFO : -8.526 per-word bound, 368.6 perplexity estimate based on a held-out corpus of 2000 documents with 261947 words\n",
      "INFO : PROGRESS: pass 2, at document #4000/4000\n",
      "INFO : merging changes from 2000 documents into a model of 4000 documents\n",
      "INFO : topic #11 (0.050): 0.011*\"JPEG\" + 0.006*\"Q\" + 0.006*\"image\" + 0.005*\"GIF\" + 0.004*\"Israel\" + 0.004*\"file\" + 0.003*\"images\" + 0.003*\"format\" + 0.003*\"free\" + 0.003*\"Jews\"\n",
      "INFO : topic #1 (0.050): 0.100*\"X\" + 0.009*\"*/\" + 0.009*\"output\" + 0.008*\"/*\" + 0.007*\"file\" + 0.006*\"*\" + 0.006*\"program\" + 0.005*\"{\" + 0.005*\"server\" + 0.005*\"Contact\"\n",
      "INFO : topic #14 (0.050): 0.013*\"o\" + 0.005*\"card\" + 0.003*\"+\" + 0.003*\"IBM\" + 0.003*\"Space\" + 0.003*\"motherboard\" + 0.002*\"mouse\" + 0.002*\"Apple\" + 0.002*\"monitor\" + 0.002*\"PC\"\n",
      "INFO : topic #4 (0.050): 0.005*\"data\" + 0.005*\"system\" + 0.004*\"software\" + 0.004*\"image\" + 0.003*\"keyboard\" + 0.003*\"netcom.com\" + 0.003*\"version\" + 0.003*\"bit\" + 0.003*\"images\" + 0.003*\"available\"\n",
      "INFO : topic #19 (0.050): 0.102*\"#\" + 0.004*\"water\" + 0.002*\"Windows\" + 0.002*\"/\" + 0.002*\"No\" + 0.002*\"read\" + 0.002*\"Mac\" + 0.002*\"please\" + 0.002*\"version\" + 0.002*\"she\"\n",
      "INFO : topic diff=1.060858, rho=0.447214\n",
      "INFO : PROGRESS: pass 3, at document #2000/4000\n",
      "INFO : merging changes from 2000 documents into a model of 4000 documents\n",
      "INFO : topic #18 (0.050): 0.007*\"window\" + 0.005*\"Windows\" + 0.004*\"Computer\" + 0.004*\"drive\" + 0.004*\"program\" + 0.003*\"font\" + 0.003*\"please\" + 0.003*\"Keywords\" + 0.003*\"8\" + 0.003*\"problems\"\n",
      "INFO : topic #15 (0.050): 0.004*\"information\" + 0.003*\"Windows\" + 0.003*\"internet\" + 0.003*\"address\" + 0.003*\"science\" + 0.002*\"example\" + 0.002*\"mail\" + 0.002*\"list\" + 0.002*\"Science\" + 0.002*\"name\"\n",
      "INFO : topic #19 (0.050): 0.121*\"#\" + 0.005*\"water\" + 0.003*\"/\" + 0.002*\"No\" + 0.002*\"read\" + 0.002*\"theism\" + 0.002*\"please\" + 0.002*\"system\" + 0.002*\"Windows\" + 0.002*\"Mac\"\n",
      "INFO : topic #13 (0.050): 0.004*\"Koresh\" + 0.003*\"Brian\" + 0.003*\"opinions\" + 0.003*\"government\" + 0.003*\"Case\" + 0.003*\"USA\" + 0.003*\"Reserve\" + 0.003*\"Cleveland\" + 0.003*\"him\" + 0.003*\"John\"\n",
      "INFO : topic #16 (0.050): 0.003*\"de\" + 0.002*\"us\" + 0.002*\"little\" + 0.002*\"again\" + 0.002*\"either\" + 0.002*\"our\" + 0.002*\"him\" + 0.002*\"hard\" + 0.002*\"evidence\" + 0.002*\"must\"\n",
      "INFO : topic diff=0.816874, rho=0.408248\n",
      "INFO : -8.415 per-word bound, 341.3 perplexity estimate based on a held-out corpus of 2000 documents with 261947 words\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO : PROGRESS: pass 3, at document #4000/4000\n",
      "INFO : merging changes from 2000 documents into a model of 4000 documents\n",
      "INFO : topic #1 (0.050): 0.113*\"X\" + 0.012*\"*/\" + 0.011*\"/*\" + 0.009*\"output\" + 0.008*\"file\" + 0.008*\"*\" + 0.007*\"{\" + 0.006*\"program\" + 0.006*\"=\" + 0.005*\"server\"\n",
      "INFO : topic #10 (0.050): 0.006*\"_/\" + 0.006*\"car\" + 0.004*\"she\" + 0.003*\"He\" + 0.003*\"him\" + 0.003*\"down\" + 0.003*\"us\" + 0.003*\"her\" + 0.002*\"gun\" + 0.002*\"police\"\n",
      "INFO : topic #16 (0.050): 0.002*\"us\" + 0.002*\"little\" + 0.002*\"our\" + 0.002*\"either\" + 0.002*\"men\" + 0.002*\"must\" + 0.002*\"again\" + 0.002*\"him\" + 0.002*\"hard\" + 0.002*\"HP\"\n",
      "INFO : topic #3 (0.050): 0.038*\"*\" + 0.035*\"0\" + 0.021*\"4\" + 0.016*\"5\" + 0.015*\"6\" + 0.013*\"8\" + 0.013*\"7\" + 0.013*\"10\" + 0.011*\"=\" + 0.010*\"9\"\n",
      "INFO : topic #5 (0.050): 0.006*\"Armenian\" + 0.005*\"government\" + 0.005*\"Turkish\" + 0.004*\"Armenians\" + 0.004*\"THE\" + 0.004*\"Azerbaijan\" + 0.004*\"our\" + 0.004*\"Armenia\" + 0.003*\"Muslims\" + 0.003*\"Muslim\"\n",
      "INFO : topic diff=0.751337, rho=0.408248\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 28s, sys: 991 ms, total: 1min 29s\n",
      "Wall time: 1min 33s\n"
     ]
    }
   ],
   "source": [
    "clipped_corpus = gensim.utils.ClippedCorpus(mm_corpus, 4000)  # use fewer documents during training, LDA is slow\n",
    "# ClippedCorpus new in gensim 0.10.1\n",
    "# copy&paste it from https://github.com/piskvorky/gensim/blob/0.10.1/gensim/utils.py#L467 if necessary (or upgrade your gensim)\n",
    "%time lda_model = gensim.models.LdaModel(clipped_corpus, num_topics=20, id2word=id2word_wiki, passes=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# id2word_wiki.doc2bow(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(4, 0.47421265), (7, 0.05221449), (10, 0.38402185), (16, 0.08484518)]"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lda_model[id2word_wiki.doc2bow(data)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO : topic #0 (0.050): 0.027*\"%\" + 0.012*\"Israel\" + 0.006*\"Machines\" + 0.006*\"ED\" + 0.005*\"Istanbul\" + 0.005*\"Israeli\" + 0.004*\"New\" + 0.004*\"/|\" + 0.004*\"population\" + 0.004*\"Turkey\"\n",
      "INFO : topic #1 (0.050): 0.113*\"X\" + 0.012*\"*/\" + 0.011*\"/*\" + 0.009*\"output\" + 0.008*\"file\" + 0.008*\"*\" + 0.007*\"{\" + 0.006*\"program\" + 0.006*\"=\" + 0.005*\"server\"\n",
      "INFO : topic #2 (0.050): 0.010*\"}\" + 0.008*\"DOS\" + 0.007*\"file\" + 0.007*\"key\" + 0.006*\"{\" + 0.006*\"system\" + 0.005*\"available\" + 0.005*\"software\" + 0.005*\"code\" + 0.004*\"*\"\n",
      "INFO : topic #3 (0.050): 0.038*\"*\" + 0.035*\"0\" + 0.021*\"4\" + 0.016*\"5\" + 0.015*\"6\" + 0.013*\"8\" + 0.013*\"7\" + 0.013*\"10\" + 0.011*\"=\" + 0.010*\"9\"\n",
      "INFO : topic #4 (0.050): 0.005*\"data\" + 0.005*\"system\" + 0.004*\"image\" + 0.004*\"software\" + 0.003*\"keyboard\" + 0.003*\"netcom.com\" + 0.003*\"bit\" + 0.003*\"version\" + 0.003*\"images\" + 0.002*\"available\"\n",
      "INFO : topic #5 (0.050): 0.006*\"Armenian\" + 0.005*\"government\" + 0.005*\"Turkish\" + 0.004*\"Armenians\" + 0.004*\"THE\" + 0.004*\"Azerbaijan\" + 0.004*\"our\" + 0.004*\"Armenia\" + 0.003*\"Muslims\" + 0.003*\"Muslim\"\n",
      "INFO : topic #6 (0.050): 0.032*\"/\" + 0.018*\"\\\" + 0.011*\"_\" + 0.006*\"card\" + 0.005*\"Apr\" + 0.005*\"___\" + 0.004*\"mhz\" + 0.004*\"25\" + 0.004*\"Doug\" + 0.004*\"GMT\"\n",
      "INFO : topic #7 (0.050): 0.008*\"year\" + 0.007*\"game\" + 0.007*\"team\" + 0.005*\"players\" + 0.005*\"*\" + 0.004*\"games\" + 0.003*\"play\" + 0.003*\"win\" + 0.003*\"him\" + 0.003*\"baseball\"\n",
      "INFO : topic #8 (0.050): 0.026*\"God\" + 0.017*\"Jesus\" + 0.009*\"He\" + 0.007*\"Christian\" + 0.007*\"Bible\" + 0.006*\"Christians\" + 0.006*\"Christ\" + 0.006*\"us\" + 0.006*\"our\" + 0.005*\"His\"\n",
      "INFO : topic #9 (0.050): 0.162*\"`\" + 0.139*\"%\" + 0.117*\"#\" + 0.053*\"M\" + 0.024*\"G\" + 0.023*\"R\" + 0.019*\"=\" + 0.017*\"+\" + 0.014*\"X\" + 0.013*\"P\"\n",
      "INFO : topic #10 (0.050): 0.006*\"_/\" + 0.006*\"car\" + 0.004*\"she\" + 0.003*\"He\" + 0.003*\"him\" + 0.003*\"down\" + 0.003*\"us\" + 0.003*\"her\" + 0.002*\"gun\" + 0.002*\"police\"\n",
      "INFO : topic #11 (0.050): 0.012*\"JPEG\" + 0.007*\"image\" + 0.006*\"Q\" + 0.005*\"GIF\" + 0.005*\"Israel\" + 0.004*\"file\" + 0.004*\"images\" + 0.003*\"format\" + 0.003*\"free\" + 0.003*\"President\"\n",
      "INFO : topic #12 (0.050): 0.010*\"drive\" + 0.004*\"each\" + 0.004*\"hard\" + 0.003*\"phone\" + 0.003*\"SCSI\" + 0.003*\"gas\" + 0.003*\"Clayton\" + 0.003*\"vs\" + 0.002*\"him\" + 0.002*\"usa\"\n",
      "INFO : topic #13 (0.050): 0.004*\"Koresh\" + 0.004*\"opinions\" + 0.003*\"government\" + 0.003*\"USA\" + 0.003*\"Cleveland\" + 0.003*\"Case\" + 0.003*\"Brian\" + 0.003*\"fire\" + 0.003*\"Reserve\" + 0.003*\"John\"\n",
      "INFO : topic #14 (0.050): 0.014*\"o\" + 0.005*\"card\" + 0.004*\"+\" + 0.004*\"Space\" + 0.003*\"IBM\" + 0.003*\"motherboard\" + 0.003*\"monitor\" + 0.003*\"Apple\" + 0.003*\"mouse\" + 0.002*\"values\"\n",
      "INFO : topic #15 (0.050): 0.004*\"Windows\" + 0.004*\"information\" + 0.003*\"science\" + 0.003*\"address\" + 0.002*\"mail\" + 0.002*\"example\" + 0.002*\"prophecy\" + 0.002*\"Science\" + 0.002*\"Gordon\" + 0.002*\"internet\"\n",
      "INFO : topic #16 (0.050): 0.002*\"us\" + 0.002*\"little\" + 0.002*\"our\" + 0.002*\"either\" + 0.002*\"men\" + 0.002*\"must\" + 0.002*\"again\" + 0.002*\"him\" + 0.002*\"hard\" + 0.002*\"HP\"\n",
      "INFO : topic #17 (0.050): 0.006*\"MSG\" + 0.005*\"Jim\" + 0.003*\"food\" + 0.003*\"*\" + 0.003*\"both\" + 0.003*\"Georgia\" + 0.003*\"Michael\" + 0.003*\"gld\" + 0.002*\"read\" + 0.002*\"_|\"\n",
      "INFO : topic #18 (0.050): 0.007*\"window\" + 0.006*\"Windows\" + 0.005*\"Computer\" + 0.004*\"Keywords\" + 0.004*\"00\" + 0.004*\"drive\" + 0.004*\"please\" + 0.004*\"program\" + 0.003*\"font\" + 0.003*\"8\"\n",
      "INFO : topic #19 (0.050): 0.118*\"#\" + 0.004*\"water\" + 0.002*\"No\" + 0.002*\"/\" + 0.002*\"read\" + 0.002*\"please\" + 0.002*\"system\" + 0.002*\"version\" + 0.002*\"Novell\" + 0.002*\"she\"\n"
     ]
    }
   ],
   "source": [
    "_ = lda_model.print_topics(-1)  # print a few most important words for each LDA topic\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(20, 13923)"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lda_model.get_topics().shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "def explore_topic(topic_number, topn=25):\n",
    "    \"\"\"\n",
    "    accept a user-supplied topic number and\n",
    "    print out a formatted list of the top terms\n",
    "    \"\"\"\n",
    "        \n",
    "    print(u'{:20} {}'.format(u'term', u'frequency') + u'\\n')\n",
    "\n",
    "    for term, frequency in lda_model.show_topic(topic_number, topn=25):\n",
    "        print(u'{:20} {:.3f}'.format(term, round(frequency, 3)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "term                 frequency\n",
      "\n",
      "_/                   0.006\n",
      "car                  0.006\n",
      "she                  0.004\n",
      "He                   0.003\n",
      "him                  0.003\n",
      "down                 0.003\n",
      "us                   0.003\n",
      "her                  0.003\n",
      "gun                  0.002\n",
      "police               0.002\n",
      "+                    0.002\n",
      "day                  0.002\n",
      "says                 0.002\n",
      "went                 0.002\n",
      "our                  0.002\n",
      "i                    0.002\n",
      "When                 0.002\n",
      "cars                 0.002\n",
      "anything             0.002\n",
      "FBI                  0.002\n",
      "BATF                 0.002\n",
      "around               0.002\n",
      "engine               0.002\n",
      "look                 0.002\n",
      "told                 0.002\n"
     ]
    }
   ],
   "source": [
    "explore_topic(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "topic  0\n",
      "term                 frequency\n",
      "\n",
      "%                    0.027\n",
      "Israel               0.012\n",
      "Machines             0.006\n",
      "ED                   0.006\n",
      "Istanbul             0.005\n",
      "Israeli              0.005\n",
      "New                  0.004\n",
      "/|                   0.004\n",
      "population           0.004\n",
      "Turkey               0.004\n",
      "Jewish               0.003\n",
      "ve                   0.003\n",
      "Jews                 0.003\n",
      "Ron                  0.003\n",
      "JPL                  0.003\n",
      "kelvin.jpl.nasa.gov  0.003\n",
      "Jet                  0.003\n",
      "Propulsion           0.003\n",
      "Ankara               0.003\n",
      "part                 0.003\n",
      "ed                   0.003\n",
      "+                    0.003\n",
      "V.                   0.002\n",
      "Laboratory           0.002\n",
      "A.                   0.002\n",
      "None\n",
      "###\n",
      "\n",
      "topic  1\n",
      "term                 frequency\n",
      "\n",
      "X                    0.113\n",
      "*/                   0.012\n",
      "/*                   0.011\n",
      "output               0.009\n",
      "file                 0.008\n",
      "*                    0.008\n",
      "{                    0.007\n",
      "program              0.006\n",
      "=                    0.006\n",
      "server               0.005\n",
      "SCSI                 0.005\n",
      "Contact              0.005\n",
      "IDE                  0.005\n",
      "speed                0.004\n",
      "drive                0.004\n",
      "window               0.004\n",
      "entry                0.004\n",
      "}                    0.004\n",
      "SGI                  0.003\n",
      "info                 0.003\n",
      "**                   0.003\n",
      "ISA                  0.003\n",
      "Motif                0.003\n",
      "bus                  0.003\n",
      "controller           0.003\n",
      "None\n",
      "###\n",
      "\n",
      "topic  2\n",
      "term                 frequency\n",
      "\n",
      "}                    0.010\n",
      "DOS                  0.008\n",
      "file                 0.007\n",
      "key                  0.007\n",
      "{                    0.006\n",
      "system               0.006\n",
      "available            0.005\n",
      "software             0.005\n",
      "code                 0.005\n",
      "*                    0.004\n",
      "chip                 0.004\n",
      "program              0.004\n",
      "Windows              0.004\n",
      "=                    0.004\n",
      "files                0.004\n",
      "Mac                  0.004\n",
      "PC                   0.003\n",
      "encryption           0.003\n",
      "number               0.003\n",
      "message              0.003\n",
      "information          0.003\n",
      "version              0.003\n",
      "keys                 0.003\n",
      "line                 0.003\n",
      "data                 0.003\n",
      "None\n",
      "###\n",
      "\n",
      "topic  3\n",
      "term                 frequency\n",
      "\n",
      "*                    0.038\n",
      "0                    0.035\n",
      "4                    0.021\n",
      "5                    0.016\n",
      "6                    0.015\n",
      "8                    0.013\n",
      "7                    0.013\n",
      "10                   0.013\n",
      "=                    0.011\n",
      "9                    0.010\n",
      "14                   0.009\n",
      "16                   0.008\n",
      "13                   0.008\n",
      "12                   0.008\n",
      "15                   0.008\n",
      "11                   0.008\n",
      "20                   0.007\n",
      "18                   0.007\n",
      "}                    0.006\n",
      "21                   0.006\n",
      "22                   0.005\n",
      "25                   0.005\n",
      "26                   0.005\n",
      "23                   0.005\n",
      "17                   0.005\n",
      "None\n",
      "###\n",
      "\n",
      "topic  4\n",
      "term                 frequency\n",
      "\n",
      "data                 0.005\n",
      "system               0.005\n",
      "image                0.004\n",
      "software             0.004\n",
      "keyboard             0.003\n",
      "netcom.com           0.003\n",
      "bit                  0.003\n",
      "version              0.003\n",
      "images               0.003\n",
      "available            0.002\n",
      "etc                  0.002\n",
      "enough               0.002\n",
      "Online               0.002\n",
      "X-Newsreader         0.002\n",
      "+                    0.002\n",
      "8                    0.002\n",
      "1.1                  0.002\n",
      "information          0.002\n",
      "Communications       0.002\n",
      "control              0.002\n",
      "access.digex.net     0.002\n",
      "card                 0.002\n",
      "color                0.002\n",
      "5                    0.002\n",
      "24                   0.002\n",
      "None\n",
      "###\n",
      "\n",
      "topic  5\n",
      "term                 frequency\n",
      "\n",
      "Armenian             0.006\n",
      "government           0.005\n",
      "Turkish              0.005\n",
      "Armenians            0.004\n",
      "THE                  0.004\n",
      "Azerbaijan           0.004\n",
      "our                  0.004\n",
      "Armenia              0.004\n",
      "Muslims              0.003\n",
      "Muslim               0.003\n",
      "against              0.003\n",
      "We                   0.003\n",
      "OF                   0.003\n",
      "dead                 0.003\n",
      "war                  0.003\n",
      "political            0.003\n",
      "President            0.002\n",
      "Serdar               0.002\n",
      "American             0.002\n",
      "Clinton              0.002\n",
      "Argic                0.002\n",
      "Russian              0.002\n",
      "rights               0.002\n",
      "law                  0.002\n",
      "Turks                0.002\n",
      "None\n",
      "###\n",
      "\n",
      "topic  6\n",
      "term                 frequency\n",
      "\n",
      "/                    0.032\n",
      "\\                    0.018\n",
      "_                    0.011\n",
      "card                 0.006\n",
      "Apr                  0.005\n",
      "___                  0.005\n",
      "mhz                  0.004\n",
      "25                   0.004\n",
      "Doug                 0.004\n",
      "GMT                  0.004\n",
      "bnr.ca               0.004\n",
      "1993                 0.003\n",
      "IIsi                 0.003\n",
      "operational          0.003\n",
      "filename             0.003\n",
      "de                   0.003\n",
      "Nick                 0.003\n",
      "}                    0.003\n",
      "__                   0.003\n",
      "{                    0.003\n",
      "David                0.002\n",
      "No                   0.002\n",
      "CPU                  0.002\n",
      "Windows              0.002\n",
      "Mark                 0.002\n",
      "None\n",
      "###\n",
      "\n",
      "topic  7\n",
      "term                 frequency\n",
      "\n",
      "year                 0.008\n",
      "game                 0.007\n",
      "team                 0.007\n",
      "players              0.005\n",
      "*                    0.005\n",
      "games                0.004\n",
      "play                 0.003\n",
      "win                  0.003\n",
      "him                  0.003\n",
      "baseball             0.003\n",
      "hit                  0.003\n",
      "hockey               0.003\n",
      "season               0.003\n",
      "Kuwait               0.003\n",
      "He                   0.002\n",
      "Mark                 0.002\n",
      "best                 0.002\n",
      "teams                0.002\n",
      "average              0.002\n",
      "lot                  0.002\n",
      "runs                 0.002\n",
      "fans                 0.002\n",
      "NHL                  0.002\n",
      "clutch               0.002\n",
      "put                  0.002\n",
      "None\n",
      "###\n",
      "\n",
      "topic  8\n",
      "term                 frequency\n",
      "\n",
      "God                  0.026\n",
      "Jesus                0.017\n",
      "He                   0.009\n",
      "Christian            0.007\n",
      "Bible                0.007\n",
      "Christians           0.006\n",
      "Christ               0.006\n",
      "us                   0.006\n",
      "our                  0.006\n",
      "His                  0.005\n",
      "human                0.005\n",
      "love                 0.004\n",
      "Christianity         0.004\n",
      "sin                  0.004\n",
      "Matthew              0.004\n",
      "faith                0.004\n",
      "him                  0.004\n",
      "life                 0.004\n",
      "book                 0.003\n",
      "appears              0.003\n",
      "man                  0.003\n",
      "We                   0.003\n",
      "read                 0.003\n",
      "John                 0.003\n",
      "must                 0.003\n",
      "None\n",
      "###\n",
      "\n",
      "topic  9\n",
      "term                 frequency\n",
      "\n",
      "`                    0.162\n",
      "%                    0.139\n",
      "#                    0.117\n",
      "M                    0.053\n",
      "G                    0.024\n",
      "R                    0.023\n",
      "=                    0.019\n",
      "+                    0.017\n",
      "X                    0.014\n",
      "P                    0.013\n",
      "N                    0.013\n",
      "B                    0.013\n",
      "Q                    0.013\n",
      "K                    0.012\n",
      "0                    0.012\n",
      "F                    0.011\n",
      "7                    0.011\n",
      "*                    0.010\n",
      "6                    0.010\n",
      "L                    0.009\n",
      "5                    0.009\n",
      "W                    0.009\n",
      "S                    0.009\n",
      "\\                    0.009\n",
      "U                    0.008\n",
      "None\n",
      "###\n",
      "\n",
      "topic  10\n",
      "term                 frequency\n",
      "\n",
      "_/                   0.006\n",
      "car                  0.006\n",
      "she                  0.004\n",
      "He                   0.003\n",
      "him                  0.003\n",
      "down                 0.003\n",
      "us                   0.003\n",
      "her                  0.003\n",
      "gun                  0.002\n",
      "police               0.002\n",
      "+                    0.002\n",
      "day                  0.002\n",
      "says                 0.002\n",
      "went                 0.002\n",
      "our                  0.002\n",
      "i                    0.002\n",
      "When                 0.002\n",
      "cars                 0.002\n",
      "anything             0.002\n",
      "FBI                  0.002\n",
      "BATF                 0.002\n",
      "around               0.002\n",
      "engine               0.002\n",
      "look                 0.002\n",
      "told                 0.002\n",
      "None\n",
      "###\n",
      "\n",
      "topic  11\n",
      "term                 frequency\n",
      "\n",
      "JPEG                 0.012\n",
      "image                0.007\n",
      "Q                    0.006\n",
      "GIF                  0.005\n",
      "Israel               0.005\n",
      "file                 0.004\n",
      "images               0.004\n",
      "format               0.003\n",
      "free                 0.003\n",
      "President            0.003\n",
      "Jews                 0.003\n",
      "We                   0.003\n",
      "version              0.003\n",
      "quality              0.003\n",
      "system               0.002\n",
      "made                 0.002\n",
      "us                   0.002\n",
      "law                  0.002\n",
      "government           0.002\n",
      "whether              0.002\n",
      "both                 0.002\n",
      "mean                 0.002\n",
      "moral                0.002\n",
      "come                 0.002\n",
      "files                0.002\n",
      "None\n",
      "###\n",
      "\n",
      "topic  12\n",
      "term                 frequency\n",
      "\n",
      "drive                0.010\n",
      "each                 0.004\n",
      "hard                 0.004\n",
      "phone                0.003\n",
      "SCSI                 0.003\n",
      "gas                  0.003\n",
      "Clayton              0.003\n",
      "vs                   0.003\n",
      "him                  0.002\n",
      "usa                  0.002\n",
      "David                0.002\n",
      "When                 0.002\n",
      "line                 0.002\n",
      "system               0.002\n",
      "three                0.002\n",
      "Do                   0.002\n",
      "power                0.002\n",
      "State                0.002\n",
      "#                    0.002\n",
      "11                   0.002\n",
      "seen                 0.002\n",
      "old                  0.002\n",
      "condition            0.002\n",
      "4                    0.002\n",
      "Ed                   0.002\n",
      "None\n",
      "###\n",
      "\n",
      "topic  13\n",
      "term                 frequency\n",
      "\n",
      "Koresh               0.004\n",
      "opinions             0.004\n",
      "government           0.003\n",
      "USA                  0.003\n",
      "Cleveland            0.003\n",
      "Case                 0.003\n",
      "Brian                0.003\n",
      "fire                 0.003\n",
      "Reserve              0.003\n",
      "John                 0.003\n",
      "him                  0.003\n",
      "David                0.003\n",
      "cleveland.Freenet.Edu 0.003\n",
      "Western              0.003\n",
      "FBI                  0.002\n",
      "wrong                0.002\n",
      "system               0.002\n",
      "nyx.cs.du.edu        0.002\n",
      "day                  0.002\n",
      "religion             0.002\n",
      "Department           0.002\n",
      "New                  0.002\n",
      "vice.ICO.TEK.COM     0.002\n",
      "tank                 0.002\n",
      "cult                 0.002\n",
      "None\n",
      "###\n",
      "\n",
      "topic  14\n",
      "term                 frequency\n",
      "\n",
      "o                    0.014\n",
      "card                 0.005\n",
      "+                    0.004\n",
      "Space                0.004\n",
      "IBM                  0.003\n",
      "motherboard          0.003\n",
      "monitor              0.003\n",
      "Apple                0.003\n",
      "mouse                0.003\n",
      "values               0.002\n",
      "power                0.002\n",
      "PC                   0.002\n",
      "Research             0.002\n",
      "VGA                  0.002\n",
      "signal               0.002\n",
      "Austin               0.002\n",
      "data                 0.002\n",
      "sound                0.002\n",
      "objective            0.002\n",
      "tower                0.002\n",
      "bus                  0.002\n",
      "available            0.002\n",
      "Computer             0.002\n",
      "both                 0.002\n",
      "case                 0.002\n",
      "None\n",
      "###\n",
      "\n",
      "topic  15\n",
      "term                 frequency\n",
      "\n",
      "Windows              0.004\n",
      "information          0.004\n",
      "science              0.003\n",
      "address              0.003\n",
      "mail                 0.002\n",
      "example              0.002\n",
      "prophecy             0.002\n",
      "Science              0.002\n",
      "Gordon               0.002\n",
      "internet             0.002\n",
      "name                 0.002\n",
      "Computer             0.002\n",
      "between              0.002\n",
      "group                0.002\n",
      "list                 0.002\n",
      "real                 0.002\n",
      "3D                   0.002\n",
      "Some                 0.002\n",
      "email                0.002\n",
      "Univ                 0.002\n",
      "Banks                0.002\n",
      "Michael              0.002\n",
      "another              0.002\n",
      "whether              0.002\n",
      "always               0.002\n",
      "None\n",
      "###\n",
      "\n",
      "topic  16\n",
      "term                 frequency\n",
      "\n",
      "us                   0.002\n",
      "little               0.002\n",
      "our                  0.002\n",
      "either               0.002\n",
      "men                  0.002\n",
      "must                 0.002\n",
      "again                0.002\n",
      "him                  0.002\n",
      "hard                 0.002\n",
      "HP                   0.002\n",
      "evidence             0.002\n",
      "de                   0.002\n",
      "mean                 0.002\n",
      "between              0.002\n",
      "GM                   0.002\n",
      "No                   0.002\n",
      "We                   0.002\n",
      "long                 0.002\n",
      "post                 0.002\n",
      "probably             0.002\n",
      "Murray               0.002\n",
      "fact                 0.002\n",
      "seems                0.002\n",
      "found                0.002\n",
      "start                0.002\n",
      "None\n",
      "###\n",
      "\n",
      "topic  17\n",
      "term                 frequency\n",
      "\n",
      "MSG                  0.006\n",
      "Jim                  0.005\n",
      "food                 0.003\n",
      "*                    0.003\n",
      "both                 0.003\n",
      "Georgia              0.003\n",
      "Michael              0.003\n",
      "gld                  0.003\n",
      "read                 0.002\n",
      "_|                   0.002\n",
      "Research             0.002\n",
      "without              0.002\n",
      "TIFF                 0.002\n",
      "Stephen              0.002\n",
      "**                   0.002\n",
      "THE                  0.002\n",
      "YES                  0.002\n",
      "having               0.002\n",
      "mcovingt             0.002\n",
      "Covington            0.002\n",
      "fact                 0.002\n",
      "Athens               0.002\n",
      "made                 0.002\n",
      "Canadian             0.002\n",
      "image                0.002\n",
      "None\n",
      "###\n",
      "\n",
      "topic  18\n",
      "term                 frequency\n",
      "\n",
      "window               0.007\n",
      "Windows              0.006\n",
      "Computer             0.005\n",
      "Keywords             0.004\n",
      "00                   0.004\n",
      "drive                0.004\n",
      "please               0.004\n",
      "program              0.004\n",
      "font                 0.003\n",
      "8                    0.003\n",
      "fonts                0.003\n",
      "Hi                   0.003\n",
      "Any                  0.003\n",
      "Science              0.003\n",
      "problems             0.003\n",
      "advance              0.003\n",
      "looking              0.003\n",
      "Does                 0.003\n",
      "Please               0.003\n",
      "3.1                  0.003\n",
      "memory               0.003\n",
      "Microsoft            0.003\n",
      "printer              0.002\n",
      "Institute            0.002\n",
      "lines                0.002\n",
      "None\n",
      "###\n",
      "\n",
      "topic  19\n",
      "term                 frequency\n",
      "\n",
      "#                    0.118\n",
      "water                0.004\n",
      "No                   0.002\n",
      "/                    0.002\n",
      "read                 0.002\n",
      "please               0.002\n",
      "system               0.002\n",
      "version              0.002\n",
      "Novell               0.002\n",
      "she                  0.002\n",
      "children             0.002\n",
      "Bill                 0.002\n",
      "She                  0.002\n",
      "Apr                  0.002\n",
      "Mac                  0.002\n",
      "Duo                  0.002\n",
      "netcom.com           0.002\n",
      "GMT                  0.002\n",
      "Windows              0.002\n",
      "run                  0.002\n",
      "anything             0.002\n",
      "CA                   0.002\n",
      "else                 0.002\n",
      "Supports             0.001\n",
      "Computer             0.001\n",
      "None\n",
      "###\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for topic in range(20):\n",
    "    print(\"topic \", topic)\n",
    "    print(explore_topic(topic_number=topic, topn=10))\n",
    "    print('###')\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help on function prepare in module pyLDAvis.gensim:\n",
      "\n",
      "prepare(topic_model, corpus, dictionary, doc_topic_dist=None, **kwargs)\n",
      "    Transforms the Gensim TopicModel and related corpus and dictionary into\n",
      "    the data structures needed for the visualization.\n",
      "    \n",
      "    Parameters\n",
      "    ----------\n",
      "    topic_model : gensim.models.ldamodel.LdaModel\n",
      "        An already trained Gensim LdaModel. The other gensim model types are\n",
      "        not supported (PRs welcome).\n",
      "    \n",
      "    corpus : array-like list of bag of word docs in tuple form or scipy CSC matrix\n",
      "        The corpus in bag of word form, the same docs used to train the model.\n",
      "        The corpus is transformed into a csc matrix internally, if you intend to\n",
      "        call prepare multiple times it is a good idea to first call\n",
      "        `gensim.matutils.corpus2csc(corpus)` and pass in the csc matrix instead.\n",
      "    \n",
      "    For example: [(50, 3), (63, 5), ....]\n",
      "    \n",
      "    dictionary: gensim.corpora.Dictionary\n",
      "        The dictionary object used to create the corpus. Needed to extract the\n",
      "        actual terms (not ids).\n",
      "    \n",
      "    doc_topic_dist (optional): Document topic distribution from LDA (default=None)\n",
      "        The document topic distribution that is eventually visualised, if you will\n",
      "        be calling `prepare` multiple times it's a good idea to explicitly pass in\n",
      "        `doc_topic_dist` as inferring this for large corpora can be quite\n",
      "        expensive.\n",
      "    \n",
      "    **kwargs :\n",
      "        additional keyword arguments are passed through to :func:`pyldavis.prepare`.\n",
      "    \n",
      "    Returns\n",
      "    -------\n",
      "    prepared_data : PreparedData\n",
      "        the data structures used in the visualization\n",
      "    \n",
      "    Example\n",
      "    --------\n",
      "    For example usage please see this notebook:\n",
      "    http://nbviewer.ipython.org/github/bmabey/pyLDAvis/blob/master/notebooks/Gensim%20Newsgroup.ipynb\n",
      "    \n",
      "    See\n",
      "    ------\n",
      "    See `pyLDAvis.prepare` for **kwargs.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "help(pyLDAvis.gensim.prepare)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/aliosha/Envs/nlp/lib/python3.6/site-packages/pyLDAvis/_prepare.py:387: DeprecationWarning: \n",
      ".ix is deprecated. Please use\n",
      ".loc for label based indexing or\n",
      ".iloc for positional indexing\n",
      "\n",
      "See the documentation here:\n",
      "http://pandas.pydata.org/pandas-docs/stable/indexing.html#ix-indexer-is-deprecated\n",
      "  topic_term_dists = topic_term_dists.ix[topic_order]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.32 s, sys: 213 ms, total: 1.53 s\n",
      "Wall time: 2.89 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "LDAvis_prepared = pyLDAvis.gensim.prepare(lda_model,list(corpus), id2word_wiki, mds='mmds')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PreparedData(topic_coordinates=       Freq  cluster  topics         x         y\n",
       "topic                                           \n",
       "0       NaN        1       1  0.185597  0.186639\n",
       "1       NaN        1       2  0.164413 -0.299004\n",
       "2       NaN        1       3  0.066456 -0.261018\n",
       "3       NaN        1       4 -0.025826  0.392700\n",
       "4       NaN        1       5  0.059392 -0.150057\n",
       "5       NaN        1       6  0.222772 -0.021026\n",
       "6       NaN        1       7  0.028403  0.192934\n",
       "7       NaN        1       8 -0.146845 -0.171723\n",
       "8       NaN        1       9 -0.309961  0.111110\n",
       "9       NaN        1      10  0.419237  0.318144\n",
       "10      NaN        1      11 -0.003074  0.017616\n",
       "11      NaN        1      12 -0.100099  0.104010\n",
       "12      NaN        1      13 -0.197983 -0.132421\n",
       "13      NaN        1      14 -0.182836  0.166872\n",
       "14      NaN        1      15 -0.051884 -0.248856\n",
       "15      NaN        1      16 -0.133572  0.003034\n",
       "16      NaN        1      17 -0.069915 -0.057487\n",
       "17      NaN        1      18 -0.267166 -0.063070\n",
       "18      NaN        1      19  0.176410 -0.141501\n",
       "19      NaN        1      20  0.166481  0.053103, topic_info=     Category  Freq            Term  Total  loglift  logprob\n",
       "term                                                        \n",
       "0     Default   0.0              12    0.0     30.0  30.0000\n",
       "1     Default   0.0        Actually    0.0     29.0  29.0000\n",
       "2     Default   0.0          Bowman    0.0     28.0  28.0000\n",
       "3     Default   0.0        Carnegie    0.0     27.0  27.0000\n",
       "4     Default   0.0          Devils    0.0     26.0  26.0000\n",
       "5     Default   0.0              He    0.0     25.0  25.0000\n",
       "6     Default   0.0         However    0.0     24.0  24.0000\n",
       "7     Default   0.0       Islanders    0.0     23.0  23.0000\n",
       "8     Default   0.0            Jagr    0.0     22.0  22.0000\n",
       "9     Default   0.0          Jersey    0.0     21.0  21.0000\n",
       "10    Default   0.0             Man    0.0     20.0  20.0000\n",
       "11    Default   0.0          Mellon    0.0     19.0  19.0000\n",
       "12    Default   0.0          Office    0.0     18.0  18.0000\n",
       "13    Default   0.0              PA    0.0     17.0  17.0000\n",
       "14    Default   0.0            Pens    0.0     16.0  16.0000\n",
       "15    Default   0.0      Pittsburgh    0.0     15.0  15.0000\n",
       "16    Default   0.0            Post    0.0     14.0  14.0000\n",
       "17    Default   0.0  andrew.cmu.edu    0.0     13.0  13.0000\n",
       "18    Default   0.0          anyway    0.0     12.0  12.0000\n",
       "19    Default   0.0            beat    0.0     11.0  11.0000\n",
       "20    Default   0.0             bit    0.0     10.0  10.0000\n",
       "21    Default   0.0        confused    0.0      9.0   9.0000\n",
       "22    Default   0.0          couple    0.0      8.0   8.0000\n",
       "23    Default   0.0    disappointed    0.0      7.0   7.0000\n",
       "24    Default   0.0             end    0.0      6.0   6.0000\n",
       "25    Default   0.0            fans    0.0      5.0   5.0000\n",
       "26    Default   0.0           final    0.0      4.0   4.0000\n",
       "27    Default   0.0             fun    0.0      3.0   3.0000\n",
       "28    Default   0.0            game    0.0      2.0   2.0000\n",
       "29    Default   0.0           games    0.0      1.0   1.0000\n",
       "...       ...   ...             ...    ...      ...      ...\n",
       "0     Topic20   0.0              12    0.0      NaN  -7.3300\n",
       "1     Topic20   0.0        Actually    0.0      NaN  -8.3575\n",
       "2     Topic20   0.0          Bowman    0.0      NaN -12.5740\n",
       "3     Topic20   0.0        Carnegie    0.0      NaN  -6.9975\n",
       "4     Topic20   0.0          Devils    0.0      NaN -12.5487\n",
       "5     Topic20   0.0              He    0.0      NaN  -7.7850\n",
       "6     Topic20   0.0         However    0.0      NaN  -7.3579\n",
       "7     Topic20   0.0       Islanders    0.0      NaN -12.4147\n",
       "8     Topic20   0.0            Jagr    0.0      NaN -12.5458\n",
       "9     Topic20   0.0          Jersey    0.0      NaN  -7.5971\n",
       "10    Topic20   0.0             Man    0.0      NaN -11.8797\n",
       "11    Topic20   0.0          Mellon    0.0      NaN  -6.9950\n",
       "12    Topic20   0.0          Office    0.0      NaN -12.1338\n",
       "13    Topic20   0.0              PA    0.0      NaN  -6.6750\n",
       "14    Topic20   0.0            Pens    0.0      NaN -12.5740\n",
       "15    Topic20   0.0      Pittsburgh    0.0      NaN  -7.2875\n",
       "16    Topic20   0.0            Post    0.0      NaN -12.1815\n",
       "17    Topic20   0.0  andrew.cmu.edu    0.0      NaN  -7.2182\n",
       "18    Topic20   0.0          anyway    0.0      NaN  -7.6140\n",
       "19    Topic20   0.0            beat    0.0      NaN -11.0876\n",
       "20    Topic20   0.0             bit    0.0      NaN  -7.2876\n",
       "21    Topic20   0.0        confused    0.0      NaN -12.5593\n",
       "22    Topic20   0.0          couple    0.0      NaN  -7.6758\n",
       "23    Topic20   0.0    disappointed    0.0      NaN -12.3930\n",
       "24    Topic20   0.0             end    0.0      NaN  -7.9587\n",
       "25    Topic20   0.0            fans    0.0      NaN  -9.6178\n",
       "26    Topic20   0.0           final    0.0      NaN  -8.2126\n",
       "27    Topic20   0.0             fun    0.0      NaN -10.1167\n",
       "28    Topic20   0.0            game    0.0      NaN  -9.5800\n",
       "29    Topic20   0.0           games    0.0      NaN -12.1079\n",
       "\n",
       "[630 rows x 6 columns], token_table=Empty DataFrame\n",
       "Columns: [Topic, Freq, Term]\n",
       "Index: [], R=30, lambda_step=0.01, plot_opts={'xlab': 'PC1', 'ylab': 'PC2'}, topic_order=[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20])"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "LDAvis_prepared"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pyLDAvis.display(LDAvis_prepared)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}