Scripts/topic_modelling.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import datetime\n",
    "from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer\n",
    "import re\n",
    "import spacy\n",
    "from spacy import displacy\n",
    "from collections import Counter\n",
    "import en_core_web_sm\n",
    "from pprint import pprint\n",
    "# gensim\n",
    "import gensim\n",
    "from gensim.corpora import Dictionary\n",
    "from gensim import models\n",
    "from gensim.parsing.preprocessing import STOPWORDS\n",
    "\n",
    "# plotting tools\n",
    "import pyLDAvis\n",
    "import pyLDAvis.gensim \n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "# nltk\n",
    "import nltk\n",
    "from nltk.tokenize import sent_tokenize\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.stem import SnowballStemmer\n",
    "from nltk.tag import pos_tag\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from nltk.stem.porter import *\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>source</th>\n",
       "      <th>headline</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2008-10-14</td>\n",
       "      <td>business_daily</td>\n",
       "      <td>Trade financing drought hits Kenyan companies</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2008-10-14</td>\n",
       "      <td>business_daily</td>\n",
       "      <td>House asks Treasury to table fresh budget pla</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2008-10-14</td>\n",
       "      <td>business_daily</td>\n",
       "      <td>Kenya gets harsh human rights verdic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2008-10-14</td>\n",
       "      <td>business_daily</td>\n",
       "      <td>Ripple effects of global crisis set to hit Eas...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2008-10-14</td>\n",
       "      <td>business_daily</td>\n",
       "      <td>Manufacturers cut product sizes to beat recessio</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         date          source  \\\n",
       "0  2008-10-14  business_daily   \n",
       "1  2008-10-14  business_daily   \n",
       "2  2008-10-14  business_daily   \n",
       "3  2008-10-14  business_daily   \n",
       "4  2008-10-14  business_daily   \n",
       "\n",
       "                                            headline  \n",
       "0      Trade financing drought hits Kenyan companies  \n",
       "1      House asks Treasury to table fresh budget pla  \n",
       "2               Kenya gets harsh human rights verdic  \n",
       "3  Ripple effects of global crisis set to hit Eas...  \n",
       "4   Manufacturers cut product sizes to beat recessio  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# topic modelling\n",
    "news = pd.read_csv(\"Cleaned Data/headlines.csv\", encoding = \"ISO-8859-1\")\n",
    "news.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0        trade financing drought hits kenyan companies\n",
       "1        house asks treasury to table fresh budget pla\n",
       "2                 kenya gets harsh human rights verdic\n",
       "3    ripple effects of global crisis set to hit eas...\n",
       "4     manufacturers cut product sizes to beat recessio\n",
       "Name: headline1, dtype: object"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "news[\"headline1\"] = news[\"headline\"].str.lower()\n",
    "news[\"headline1\"] = news[\"headline1\"].str.replace(\"[^a-z\\\\s]\", \" \")\n",
    "news[\"headline1\"] = news[\"headline1\"].str.replace(\"\\\\s+\", \" \")\n",
    "news[\"headline1\"].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    [trade, financing, drought, hits, kenyan, comp...\n",
       "1    [house, asks, treasury, to, table, fresh, budg...\n",
       "2          [kenya, gets, harsh, human, rights, verdic]\n",
       "3    [ripple, effects, of, global, crisis, set, to,...\n",
       "4    [manufacturers, cut, product, sizes, to, beat,...\n",
       "Name: headline1, dtype: object"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# tokenization\n",
    "news[\"headline1\"] = news[\"headline1\"].apply(word_tokenize)\n",
    "news[\"headline1\"].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    [trade, financing, drought, hit, kenyan, company]\n",
       "1    [house, asks, treasury, to, table, fresh, budg...\n",
       "2            [kenya, get, harsh, human, right, verdic]\n",
       "3    [ripple, effect, of, global, crisis, set, to, ...\n",
       "4    [manufacturer, cut, product, size, to, beat, r...\n",
       "Name: headline1, dtype: object"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lemmatizer = WordNetLemmatizer()\n",
    "news[\"headline1\"] = news[\"headline1\"].apply(lambda x: [lemmatizer.lemmatize(y) for y in x])\n",
    "news[\"headline1\"].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0            [trade, financing, drought, hit, company]\n",
       "1    [house, asks, treasury, table, fresh, budget, ...\n",
       "2                   [get, harsh, human, right, verdic]\n",
       "3    [ripple, effect, global, crisis, set, hit, eas...\n",
       "4    [manufacturer, cut, product, size, beat, reces...\n",
       "Name: headline1, dtype: object"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "english_stopwords = stopwords.words('english')\n",
    "english_stopwords.extend([\"news\", \"live blog\", \"live\",  \"in pictures\", \"download\", \"blog\",\n",
    "   \"lifestyle\", \"editorial\", \"editorials\", \"pictures\", \"opinion\", \"read\",\n",
    "  \"newsplex\", \"politics\", \"new\", \"news live\", \"photos\",  \"video\", \n",
    "  \"frontrow\", \"fact check\", \"financial time\", \"watch live\", \"columnists\", \"interview\", \"habari\", \"kenya\", \"kenyan\", \"star\"])\n",
    "news[\"headline1\"] = news[\"headline1\"].apply(lambda x: [y for y in x if y not in english_stopwords and  len(y) > 2])\n",
    "news[\"headline1\"].head()                        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create bag of words\n",
    "dictionary = Dictionary(news[\"headline1\"]) # dictionary of 51127 words\n",
    "dictionary.filter_extremes(no_below = 5, no_above = 0.5) # dictionary of 18063 words\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 company\n",
      "1 drought\n",
      "2 financing\n",
      "3 hit\n",
      "4 trade\n",
      "5 asks\n",
      "6 budget\n",
      "7 fresh\n",
      "8 house\n",
      "9 pla\n"
     ]
    }
   ],
   "source": [
    "count = 0\n",
    "for k, v in dictionary.iteritems():\n",
    "    print(k, v)\n",
    "    count += 1\n",
    "    if count == 10:\n",
    "        break\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'company'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bow = [dictionary.doc2bow(headline) for headline in news[\"headline1\"]]\n",
    "dictionary[bow[0][0][0]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "lda_model = gensim.models.LdaMulticore(bow, num_topics=3, id2word=dictionary, passes=2, workers=2, per_word_topics = True, random_state = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(0,\n",
      "  '0.008*\"market\" + 0.008*\"firm\" + 0.007*\"bank\" + 0.007*\"company\" + '\n",
      "  '0.006*\"africa\" + 0.006*\"deal\" + 0.006*\"county\" + 0.006*\"state\" + '\n",
      "  '0.006*\"plan\" + 0.005*\"farmer\"'),\n",
      " (1,\n",
      "  '0.010*\"man\" + 0.010*\"woman\" + 0.009*\"police\" + 0.007*\"year\" + 0.006*\"two\" + '\n",
      "  '0.005*\"death\" + 0.005*\"life\" + 0.004*\"killed\" + 0.004*\"attack\" + '\n",
      "  '0.004*\"nairobi\"'),\n",
      " (2,\n",
      "  '0.012*\"uhuru\" + 0.009*\"get\" + 0.008*\"ruto\" + 0.008*\"raila\" + 0.008*\"court\" '\n",
      "  '+ 0.007*\"say\" + 0.006*\"governor\" + 0.006*\"leader\" + 0.005*\"president\" + '\n",
      "  '0.005*\"world\"')]\n"
     ]
    }
   ],
   "source": [
    "pprint(lda_model.print_topics())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Chance\\Anaconda3\\lib\\site-packages\\pyLDAvis\\_prepare.py:257: FutureWarning: Sorting because non-concatenation axis is not aligned. A future version\n",
      "of pandas will change to not sort by default.\n",
      "\n",
      "To accept the future behavior, pass 'sort=True'.\n",
      "\n",
      "To retain the current behavior and silence the warning, pass sort=False\n",
      "\n",
      "  return pd.concat([default_term_info] + list(topic_dfs))\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "<link rel=\"stylesheet\" type=\"text/css\" href=\"https://cdn.rawgit.com/bmabey/pyLDAvis/files/ldavis.v1.0.0.css\">\n",
       "\n",
       "\n",
       "<div id=\"ldavis_el1071620272854234727796538279\"></div>\n",
       "<script type=\"text/javascript\">\n",
       "\n",
       "var ldavis_el1071620272854234727796538279_data = {\"mdsDat\": {\"x\": [0.2338341567996359, -0.2819622953037225, 0.04812813850408642], \"y\": [-0.17022255299320657, -0.09576573357735316, 0.26598828657055984], \"topics\": [1, 2, 3], \"cluster\": [1, 1, 1], \"Freq\": [33.204532623291016, 32.11139678955078, 34.6840705871582]}, \"tinfo\": {\"Category\": [\"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\"], \"Freq\": [7115.0, 5531.0, 4565.0, 4645.0, 4914.0, 4672.0, 4702.0, 5743.0, 4120.0, 4038.0, 4584.0, 3714.0, 3376.0, 3411.0, 3126.0, 3692.0, 3451.0, 3043.0, 2856.0, 2746.0, 2816.0, 3146.0, 2689.0, 5218.0, 2253.0, 2789.0, 2203.0, 2145.0, 2295.0, 2236.0, 4645.06591796875, 3713.466552734375, 3375.76123046875, 4037.063720703125, 1904.5009765625, 2815.072509765625, 4118.21630859375, 1451.767578125, 2854.7802734375, 1215.805908203125, 1343.0128173828125, 1322.06494140625, 1134.8353271484375, 1031.0025634765625, 1081.2513427734375, 1036.9796142578125, 1327.5162353515625, 936.322265625, 924.8167114257812, 874.7597045898438, 818.7342529296875, 1185.560546875, 951.5938720703125, 1124.498291015625, 822.8822631835938, 821.8331909179688, 695.4534912109375, 787.5386962890625, 706.0219116210938, 1068.6044921875, 3394.724609375, 1919.273681640625, 1684.51953125, 1804.9288330078125, 1500.2471923828125, 2223.230712890625, 1752.1883544921875, 1711.231689453125, 3013.73828125, 1732.533203125, 1117.092041015625, 1290.1876220703125, 1782.7138671875, 1745.638427734375, 1365.8660888671875, 1448.3651123046875, 1474.4072265625, 2549.197998046875, 1470.859619140625, 1363.8223876953125, 3066.154052734375, 1887.4600830078125, 3346.16064453125, 1606.0260009765625, 2217.081298828125, 1913.373291015625, 1992.6688232421875, 1627.8951416015625, 1615.9190673828125, 4564.61279296875, 2252.819091796875, 2144.812255859375, 1651.373291015625, 1859.2994384765625, 1615.1597900390625, 1455.9273681640625, 1453.509033203125, 1326.734375, 2744.65771484375, 1124.3690185546875, 1045.7459716796875, 1403.93017578125, 1283.500732421875, 1713.367919921875, 1642.620361328125, 889.8444213867188, 3123.5693359375, 847.0314331054688, 1386.966552734375, 1234.101318359375, 1292.312255859375, 1203.54052734375, 821.8642578125, 815.8184204101562, 1359.49755859375, 734.6690673828125, 715.5394897460938, 814.3904418945312, 695.3650512695312, 5519.525390625, 2200.44482421875, 1596.9615478515625, 1171.689453125, 1727.142333984375, 1654.310546875, 1136.1768798828125, 1100.3311767578125, 1873.62060546875, 1265.953125, 2629.097900390625, 2108.344970703125, 5172.57275390625, 1647.409423828125, 2154.467529296875, 1991.27001953125, 1837.2537841796875, 3596.487060546875, 2096.762939453125, 1995.654541015625, 2170.56005859375, 1691.4595947265625, 1802.92724609375, 1530.9542236328125, 1866.515869140625, 1757.9483642578125, 1709.46337890625, 2293.99267578125, 1329.427734375, 1252.9432373046875, 790.0603637695312, 760.5679321289062, 938.702392578125, 1345.0457763671875, 650.4122314453125, 1830.5032958984375, 1680.46630859375, 627.3302001953125, 816.5588989257812, 1067.9512939453125, 2080.48583984375, 668.45654296875, 622.3314208984375, 801.3525390625, 774.857421875, 1916.8743896484375, 455.95849609375, 526.2890014648438, 475.2410583496094, 448.7015075683594, 1518.86767578125, 421.82073974609375, 534.9528198242188, 602.9611206054688, 7082.2900390625, 4663.6484375, 4880.80126953125, 4668.3857421875, 2095.79150390625, 2227.133544921875, 1209.4544677734375, 838.8571166992188, 905.663330078125, 839.1233520507812, 741.0891723632812, 3380.37451171875, 4442.2099609375, 1349.0245361328125, 3570.615234375, 3034.85205078125, 1444.01806640625, 2694.731689453125, 1408.9876708984375, 1641.687255859375, 1194.7322998046875, 1460.3935546875, 1644.5902099609375, 2361.507568359375, 1978.6768798828125, 1508.915771484375, 1833.859619140625, 1830.8564453125, 3794.7646484375, 2696.061279296875, 1972.087158203125, 2425.330322265625, 1776.066650390625, 2660.94921875, 1947.3116455078125, 1987.4144287109375, 1895.3658447265625, 1892.3446044921875, 1842.5770263671875], \"Term\": [\"uhuru\", \"man\", \"police\", \"market\", \"get\", \"raila\", \"ruto\", \"woman\", \"firm\", \"bank\", \"court\", \"company\", \"deal\", \"africa\", \"two\", \"governor\", \"leader\", \"plan\", \"farmer\", \"death\", \"tax\", \"president\", \"life\", \"year\", \"killed\", \"team\", \"attack\", \"arrested\", \"jubilee\", \"business\", \"market\", \"company\", \"deal\", \"bank\", \"price\", \"tax\", \"firm\", \"loan\", \"farmer\", \"profit\", \"fall\", \"maize\", \"capital\", \"growth\", \"personal\", \"finance\", \"loss\", \"tea\", \"investor\", \"economy\", \"rate\", \"share\", \"low\", \"safaricom\", \"hotel\", \"cbk\", \"nse\", \"chinese\", \"housing\", \"debt\", \"africa\", \"billion\", \"fund\", \"eye\", \"oil\", \"business\", \"launch\", \"rise\", \"plan\", \"cost\", \"kra\", \"sign\", \"cut\", \"food\", \"boost\", \"trade\", \"china\", \"hit\", \"local\", \"project\", \"state\", \"power\", \"county\", \"cash\", \"set\", \"pay\", \"nairobi\", \"million\", \"year\", \"police\", \"killed\", \"arrested\", \"dead\", \"wife\", \"murder\", \"baby\", \"officer\", \"old\", \"death\", \"charged\", \"found\", \"suspect\", \"killing\", \"trump\", \"kill\", \"husband\", \"two\", \"son\", \"love\", \"gor\", \"doctor\", \"sonko\", \"seven\", \"die\", \"crash\", \"missing\", \"shot\", \"father\", \"boda\", \"man\", \"attack\", \"men\", \"mother\", \"mombasa\", \"five\", \"coach\", \"arrest\", \"three\", \"boy\", \"life\", \"child\", \"woman\", \"fire\", \"one\", \"family\", \"girl\", \"year\", \"day\", \"city\", \"nairobi\", \"student\", \"school\", \"home\", \"poll\", \"party\", \"election\", \"jubilee\", \"odm\", \"medium\", \"referendum\", \"play\", \"champion\", \"iebc\", \"ruling\", \"teacher\", \"graft\", \"tsc\", \"vote\", \"political\", \"cup\", \"fitness\", \"opposition\", \"afcon\", \"social\", \"cancer\", \"supreme\", \"boss\", \"salary\", \"kuria\", \"meet\", \"knut\", \"senate\", \"date\", \"uhuru\", \"raila\", \"get\", \"ruto\", \"league\", \"must\", \"told\", \"eacc\", \"debate\", \"anti\", \"kalonzo\", \"leader\", \"court\", \"right\", \"governor\", \"president\", \"final\", \"team\", \"use\", \"work\", \"fake\", \"free\", \"money\", \"tell\", \"census\", \"title\", \"law\", \"order\", \"say\", \"world\", \"call\", \"want\", \"stop\", \"county\", \"case\", \"win\", \"health\", \"state\", \"school\"], \"Total\": [7115.0, 5531.0, 4565.0, 4645.0, 4914.0, 4672.0, 4702.0, 5743.0, 4120.0, 4038.0, 4584.0, 3714.0, 3376.0, 3411.0, 3126.0, 3692.0, 3451.0, 3043.0, 2856.0, 2746.0, 2816.0, 3146.0, 2689.0, 5218.0, 2253.0, 2789.0, 2203.0, 2145.0, 2295.0, 2236.0, 4645.7490234375, 3714.174072265625, 3376.6357421875, 4038.11767578125, 1905.18408203125, 2816.133056640625, 4120.10595703125, 1452.4425048828125, 2856.212890625, 1216.4815673828125, 1343.7855224609375, 1322.854736328125, 1135.537841796875, 1031.69970703125, 1081.9896240234375, 1037.706787109375, 1328.474609375, 937.0323486328125, 925.536376953125, 875.4490356445312, 819.4021606445312, 1186.54931640625, 952.4323120117188, 1125.49951171875, 823.6371459960938, 822.6013793945312, 696.1286010742188, 788.3079833984375, 706.716796875, 1069.6748046875, 3411.87451171875, 1923.054443359375, 1686.6624755859375, 1809.0721435546875, 1502.1873779296875, 2236.92529296875, 1758.9967041015625, 1717.5224609375, 3043.930419921875, 1741.0421142578125, 1118.423583984375, 1293.6488037109375, 1796.9664306640625, 1766.553466796875, 1376.46142578125, 1472.4764404296875, 1517.5218505859375, 2870.00341796875, 1525.077392578125, 1396.539306640625, 4990.71728515625, 2323.39208984375, 6039.28515625, 1858.9654541015625, 3818.7216796875, 3045.76611328125, 4164.06884765625, 2285.01171875, 5218.1328125, 4565.59130859375, 2253.55810546875, 2145.592529296875, 1652.0704345703125, 1860.115478515625, 1615.8692626953125, 1456.65869140625, 1454.2454833984375, 1327.47021484375, 2746.374267578125, 1125.0963134765625, 1046.4500732421875, 1404.9105224609375, 1284.4033203125, 1714.6583251953125, 1643.8931884765625, 890.5349731445312, 3126.17041015625, 847.7550048828125, 1388.1597900390625, 1235.1956787109375, 1293.459716796875, 1204.611328125, 822.6057739257812, 816.5564575195312, 1360.78857421875, 735.3908081054688, 716.2438354492188, 815.213134765625, 696.0934448242188, 5531.498046875, 2203.1572265625, 1598.9302978515625, 1172.9613037109375, 1736.0313720703125, 1662.4554443359375, 1138.0748291015625, 1101.9075927734375, 1887.704833984375, 1269.712158203125, 2689.367431640625, 2159.707763671875, 5743.16650390625, 1693.0887451171875, 2411.656005859375, 2275.83056640625, 2071.98828125, 5218.1328125, 3039.449951171875, 3051.982177734375, 4164.06884765625, 2492.6494140625, 3664.3642578125, 2000.01220703125, 1867.216552734375, 1758.673828125, 1710.22265625, 2295.185791015625, 1330.15625, 1253.8870849609375, 790.7482299804688, 761.290283203125, 939.5975952148438, 1346.3565673828125, 651.0847778320312, 1832.4027099609375, 1682.2177734375, 627.9886474609375, 817.422607421875, 1069.12890625, 2082.8642578125, 669.3114624023438, 623.1376953125, 802.4220581054688, 775.9595336914062, 1919.6036376953125, 456.61224365234375, 527.0631103515625, 475.9460144042969, 449.4009704589844, 1521.2557373046875, 422.4861755371094, 535.80078125, 603.9391479492188, 7115.33056640625, 4672.748046875, 4914.3837890625, 4702.2138671875, 2101.5439453125, 2243.729248046875, 1211.953857421875, 840.2911376953125, 908.4889526367188, 841.5380249023438, 742.3438720703125, 3451.2265625, 4584.1455078125, 1361.6685791015625, 3692.009033203125, 3146.4775390625, 1465.2003173828125, 2789.88525390625, 1432.8114013671875, 1684.237060546875, 1209.951171875, 1497.6851806640625, 1705.3065185546875, 2586.5751953125, 2125.538330078125, 1557.67822265625, 2099.139892578125, 2107.19287109375, 6307.5205078125, 3822.376953125, 2513.2783203125, 3678.456298828125, 2141.62451171875, 6039.28515625, 2766.38623046875, 3063.1513671875, 3040.20361328125, 4990.71728515625, 3664.3642578125], \"loglift\": [30.0, 29.0, 28.0, 27.0, 26.0, 25.0, 24.0, 23.0, 22.0, 21.0, 20.0, 19.0, 18.0, 17.0, 16.0, 15.0, 14.0, 13.0, 12.0, 11.0, 10.0, 9.0, 8.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, 1.1023000478744507, 1.1023000478744507, 1.1022000312805176, 1.1022000312805176, 1.1021000146865845, 1.1021000146865845, 1.1019999980926514, 1.1019999980926514, 1.1019999980926514, 1.1018999814987183, 1.1018999814987183, 1.1018999814987183, 1.1018999814987183, 1.1017999649047852, 1.1017999649047852, 1.1017999649047852, 1.1017999649047852, 1.101699948310852, 1.101699948310852, 1.101699948310852, 1.101699948310852, 1.1016000509262085, 1.1016000509262085, 1.1016000509262085, 1.1016000509262085, 1.1015000343322754, 1.1015000343322754, 1.1015000343322754, 1.1015000343322754, 1.1015000343322754, 1.0973999500274658, 1.1004999876022339, 1.101199984550476, 1.1002000570297241, 1.101199984550476, 1.0963000059127808, 1.0986000299453735, 1.0987999439239502, 1.0924999713897705, 1.097599983215332, 1.1013000011444092, 1.0997999906539917, 1.094499945640564, 1.0906000137329102, 1.0947999954223633, 1.0859999656677246, 1.073699951171875, 0.9839000105857849, 1.0663000345230103, 1.0787999629974365, 0.6152999997138977, 0.8946999907493591, 0.5120000243186951, 0.9562000036239624, 0.5587999820709229, 0.6376000046730042, 0.36550000309944153, 0.7634000182151794, -0.0697999969124794, 1.135699987411499, 1.135599970817566, 1.135599970817566, 1.1354999542236328, 1.1354999542236328, 1.1354999542236328, 1.1354999542236328, 1.1354999542236328, 1.1354000568389893, 1.1353000402450562, 1.1353000402450562, 1.1353000402450562, 1.1353000402450562, 1.1353000402450562, 1.135200023651123, 1.135200023651123, 1.135200023651123, 1.13510000705719, 1.13510000705719, 1.13510000705719, 1.13510000705719, 1.13510000705719, 1.13510000705719, 1.13510000705719, 1.13510000705719, 1.1349999904632568, 1.1349999904632568, 1.1349999904632568, 1.1348999738693237, 1.1348999738693237, 1.1338000297546387, 1.1346999406814575, 1.1346999406814575, 1.1348999738693237, 1.1308000087738037, 1.13100004196167, 1.1342999935150146, 1.1345000267028809, 1.128499984741211, 1.1330000162124634, 1.1132999658584595, 1.111899971961975, 1.0312999486923218, 1.1086000204086304, 1.0232000350952148, 1.0024000406265259, 1.0156999826431274, 0.7638000249862671, 0.7646999955177307, 0.7110999822616577, 0.484499990940094, 0.748199999332428, 0.42669999599456787, 0.8687000274658203, 1.058500051498413, 1.058500051498413, 1.05840003490448, 1.05840003490448, 1.0583000183105469, 1.0580999851226807, 1.0579999685287476, 1.0578999519348145, 1.0578999519348145, 1.0578999519348145, 1.0578999519348145, 1.0578999519348145, 1.057800054550171, 1.057800054550171, 1.057800054550171, 1.057800054550171, 1.0577000379562378, 1.0576000213623047, 1.0576000213623047, 1.0576000213623047, 1.0575000047683716, 1.0575000047683716, 1.0575000047683716, 1.0573999881744385, 1.0573999881744385, 1.0572999715805054, 1.0572999715805054, 1.0572999715805054, 1.0572999715805054, 1.0572999715805054, 1.0542000532150269, 1.0569000244140625, 1.0520000457763672, 1.0516999959945679, 1.0561000108718872, 1.0514999628067017, 1.0568000078201294, 1.0571999549865723, 1.055799961090088, 1.055999994277954, 1.0571999549865723, 1.038100004196167, 1.027400016784668, 1.0496000051498413, 1.0255000591278076, 1.0227999687194824, 1.0442999601364136, 1.0241999626159668, 1.042099952697754, 1.0333000421524048, 1.0462000370025635, 1.0336999893188477, 1.0226000547409058, 0.9678999781608582, 0.9872999787330627, 1.0270999670028687, 0.923799991607666, 0.9182999730110168, 0.5508000254631042, 0.7098000049591064, 0.8163999915122986, 0.6424000263214111, 0.8716999888420105, 0.23929999768733978, 0.7077999711036682, 0.6262999773025513, 0.5863999724388123, 0.08910000324249268, 0.37139999866485596], \"logprob\": [30.0, 29.0, 28.0, 27.0, 26.0, 25.0, 24.0, 23.0, 22.0, 21.0, 20.0, 19.0, 18.0, 17.0, 16.0, 15.0, 14.0, 13.0, 12.0, 11.0, 10.0, 9.0, 8.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, -4.769899845123291, -4.993800163269043, -5.089099884033203, -4.910200119018555, -5.661499977111816, -5.270699977874756, -4.8902997970581055, -5.9328999519348145, -5.256700038909912, -6.110300064086914, -6.010799884796143, -6.026500225067139, -6.179200172424316, -6.275199890136719, -6.22760009765625, -6.269400119781494, -6.02239990234375, -6.371500015258789, -6.383900165557861, -6.439499855041504, -6.50570011138916, -6.135499954223633, -6.355299949645996, -6.188399791717529, -6.500699996948242, -6.5019001960754395, -6.668900012969971, -6.544600009918213, -6.653800010681152, -6.2393999099731445, -5.083499908447266, -5.653800010681152, -5.784200191497803, -5.715199947357178, -5.900100231170654, -5.506800174713135, -5.744900226593018, -5.768499851226807, -5.202499866485596, -5.756100177764893, -6.195000171661377, -6.050899982452393, -5.72760009765625, -5.748600006103516, -5.993899822235107, -5.935299873352051, -5.917500019073486, -5.369900226593018, -5.919899940490723, -5.9953999519348145, -5.185299873352051, -5.670499801635742, -5.097899913787842, -5.831999778747559, -5.509500026702881, -5.656899929046631, -5.616199970245361, -5.818399906158447, -5.825799942016602, -4.753900051116943, -5.460100173950195, -5.509200096130371, -5.770599842071533, -5.6519999504089355, -5.792799949645996, -5.896599769592285, -5.8983001708984375, -5.989500045776367, -5.262599945068359, -6.15500020980835, -6.227499961853027, -5.933000087738037, -6.02269983291626, -5.733799934387207, -5.776000022888184, -6.388999938964844, -5.133299827575684, -6.438300132751465, -5.945099830627441, -6.0619001388549805, -6.0157999992370605, -6.086999893188477, -6.468400001525879, -6.475800037384033, -5.965099811553955, -6.580599784851074, -6.60699987411499, -6.47760009765625, -6.6356000900268555, -4.564000129699707, -5.48360013961792, -5.804100036621094, -6.113800048828125, -5.725800037384033, -5.768899917602539, -6.144599914550781, -6.176599979400635, -5.644400119781494, -6.036399841308594, -5.305600166320801, -5.526299953460693, -4.628900051116943, -5.7729997634887695, -5.504700183868408, -5.583499908447266, -5.664000034332275, -4.992300033569336, -5.531899929046631, -5.581299781799316, -5.497300148010254, -5.746699810028076, -5.682799816131592, -5.846399784088135, -5.725200176239014, -5.785200119018555, -5.8130998611450195, -5.519000053405762, -6.064599990844727, -6.123799800872803, -6.585000038146973, -6.623000144958496, -6.412600040435791, -6.0528998374938965, -6.7795000076293945, -5.744699954986572, -5.8302001953125, -6.8155999183654785, -6.552000045776367, -6.283599853515625, -5.616700172424316, -6.752099990844727, -6.823599815368652, -6.570799827575684, -6.604400157928467, -5.698599815368652, -7.134699821472168, -6.991199970245361, -7.093200206756592, -7.150700092315674, -5.931399822235107, -7.212500095367432, -6.974899768829346, -6.855199813842773, -4.39169979095459, -4.809500217437744, -4.763999938964844, -4.808499813079834, -5.609399795532227, -5.548600196838379, -6.15910005569458, -6.525000095367432, -6.448400020599365, -6.524700164794922, -6.64900016784668, -5.13129997253418, -4.8582000732421875, -6.049900054931641, -5.076600074768066, -5.239200115203857, -5.981900215148926, -5.357999801635742, -6.006400108337402, -5.853600025177002, -6.17140007019043, -5.970600128173828, -5.851799964904785, -5.489999771118164, -5.666900157928467, -5.937900066375732, -5.7428998947143555, -5.744500160217285, -5.015699863433838, -5.357500076293945, -5.670199871063232, -5.4633002281188965, -5.774899959564209, -5.37060022354126, -5.6828999519348145, -5.662499904632568, -5.70989990234375, -5.71150016784668, -5.7382001876831055]}, \"token.table\": {\"Topic\": [1, 3, 1, 2, 3, 1, 2, 3, 2, 3, 2, 1, 2, 2, 1, 3, 1, 2, 3, 2, 1, 3, 3, 1, 2, 3, 1, 2, 3, 1, 2, 3, 2, 3, 1, 1, 2, 3, 1, 2, 3, 1, 1, 2, 3, 2, 3, 2, 1, 2, 3, 1, 2, 3, 1, 1, 2, 3, 1, 2, 3, 1, 1, 3, 1, 2, 3, 1, 2, 3, 1, 2, 2, 3, 1, 3, 2, 3, 1, 2, 3, 2, 1, 1, 2, 1, 2, 3, 1, 3, 2, 1, 2, 3, 2, 3, 1, 3, 1, 2, 3, 1, 2, 3, 1, 1, 2, 3, 1, 3, 2, 1, 2, 3, 1, 1, 2, 3, 1, 3, 1, 3, 1, 2, 1, 2, 3, 2, 1, 3, 1, 3, 1, 2, 3, 2, 3, 2, 3, 1, 2, 3, 1, 3, 1, 1, 2, 3, 1, 2, 3, 1, 2, 3, 1, 1, 2, 2, 3, 1, 1, 3, 2, 3, 1, 2, 2, 2, 3, 3, 1, 3, 3, 1, 2, 3, 1, 2, 3, 1, 2, 3, 1, 2, 3, 1, 2, 3, 1, 1, 2, 3, 1, 3, 1, 2, 1, 3, 1, 1, 2, 3, 1, 3, 1, 2, 3, 2, 3, 1, 2, 3, 2, 1, 2, 3, 1, 2, 3, 2, 3, 2, 1, 3, 1, 2, 3, 1, 3, 2, 1, 2, 2, 1, 2, 3, 3, 1, 2, 3, 3, 1, 2, 3, 1, 1, 2, 3, 3, 2, 1, 2, 3, 3, 1, 2, 3, 1, 2, 3, 1, 1, 1, 3, 2, 3, 1, 3, 1, 2, 3, 1, 2, 3, 3, 1, 2, 3, 1, 2, 3, 1, 2, 3, 1, 2, 3, 1, 3, 1, 2, 3, 2, 1, 2, 2, 1, 2, 1, 3, 2, 1, 2, 1, 2, 3, 1, 2, 3, 1, 2, 3, 3, 2, 3, 1, 3, 1, 1, 2, 3, 1, 2, 3, 1, 2, 3, 1, 2, 3, 2, 3, 2, 3, 1, 3, 1, 2, 3, 3, 1, 2, 3, 1, 2, 3, 1, 3, 3, 1, 2, 3, 2, 1, 2, 3, 1, 2, 3, 1, 2, 3, 1, 2, 3, 1, 2, 3], \"Freq\": [0.0012462269514799118, 0.9982277750968933, 0.9950541853904724, 0.0005861880490556359, 0.004396410193294287, 0.002376600867137313, 0.0011883004335686564, 0.99698406457901, 0.9982688426971436, 0.0009075170964933932, 0.9997238516807556, 0.0009077881113626063, 0.9985669255256653, 0.9995477795600891, 0.9997231960296631, 0.0002476401277817786, 0.9978916645050049, 0.0005200060550123453, 0.0015600182814523578, 0.9984291791915894, 0.9923997521400452, 0.007265005726367235, 0.9979829788208008, 0.0007875800947658718, 0.9970763921737671, 0.0023627402260899544, 0.9937748312950134, 0.0026822532527148724, 0.0031292953062802553, 0.18183422088623047, 0.03342248126864433, 0.7846325635910034, 0.0010418817400932312, 0.9986436367034912, 0.9995263814926147, 0.001445929752662778, 0.2946081757545471, 0.7038062810897827, 0.8639214038848877, 0.0026896679773926735, 0.13286960124969482, 0.9992689490318298, 0.0009409380727447569, 0.06821801513433456, 0.9310582280158997, 0.0010642854031175375, 0.9993639588356018, 0.9990255832672119, 0.001852102461270988, 0.9760580062866211, 0.02222522906959057, 0.9713204503059387, 0.018451133742928505, 0.009884536266326904, 0.9996092915534973, 0.34534934163093567, 0.6540011763572693, 0.0006553118000738323, 0.0008786768303252757, 0.9981768727302551, 0.0008786768303252757, 0.9996839165687561, 0.9953808784484863, 0.004594949074089527, 0.5540390610694885, 0.005298640578985214, 0.44061505794525146, 0.0006544294883497059, 0.03032189980149269, 0.9689919352531433, 0.0007348679937422276, 0.9986856579780579, 0.0009602162172086537, 0.998624861240387, 0.9922277927398682, 0.0077909077517688274, 0.0016557959606871009, 0.9984449744224548, 0.0009870206704363227, 0.6899274587631226, 0.30926647782325745, 0.9993520379066467, 0.9998117089271545, 0.00036411642213352025, 0.9994996190071106, 0.002201457740738988, 0.001100728870369494, 0.997260332107544, 0.9993691444396973, 0.0009348635794594884, 0.999318540096283, 0.0007731203222647309, 0.9988714456558228, 0.0007731203222647309, 0.001190063776448369, 0.9984634518623352, 0.9994871020317078, 0.999285101890564, 0.9977490305900574, 0.0016583086689934134, 0.0005527695757336915, 0.001652959268540144, 0.01074423547834158, 0.9876431822776794, 0.9994154572486877, 0.12347140908241272, 0.8748454451560974, 0.0013182000257074833, 0.9995753765106201, 0.00035011395812034607, 0.9985118508338928, 0.001365001080557704, 0.012967510148882866, 0.9855307936668396, 0.9993188977241516, 0.0035438190679997206, 0.9727783203125, 0.023625459522008896, 0.9994888305664062, 0.000485424418002367, 0.0014940727269276977, 0.9980405569076538, 0.004812159109860659, 0.9949138760566711, 0.9883652329444885, 0.008491110056638718, 0.003396444022655487, 0.9995698928833008, 0.024704791605472565, 0.9748377203941345, 0.9990143179893494, 0.0011857737554237247, 0.005290592089295387, 0.0014243901241570711, 0.9932069182395935, 0.8865880370140076, 0.11293499916791916, 0.9990320205688477, 0.0008095883531495929, 0.00785480160266161, 0.025189537554979324, 0.9672240614891052, 0.0005944533622823656, 0.9986816644668579, 0.9993218183517456, 0.37102776765823364, 0.005591730587184429, 0.6233134865760803, 0.8881522417068481, 0.10592322051525116, 0.0055749062448740005, 0.22949859499931335, 0.7654953002929688, 0.004999969620257616, 0.999226450920105, 0.9989857077598572, 0.9993992447853088, 0.000742745294701308, 0.9989924430847168, 0.9994204640388489, 0.0004356945864856243, 0.9994833469390869, 0.0013470846461132169, 0.9981896877288818, 0.000608312024269253, 0.9994566440582275, 0.9997523427009583, 0.9996860027313232, 0.0007785716443322599, 0.9988492727279663, 0.9987271428108215, 0.0008941155974753201, 0.9991077780723572, 0.9960223436355591, 0.0005685059004463255, 0.002842529444023967, 0.12243109941482544, 0.0038110846653580666, 0.873691201210022, 0.0005795041215606034, 0.0199928916990757, 0.9793619513511658, 0.0009516812860965729, 0.0019033625721931458, 0.997361958026886, 0.0007436692831106484, 0.9775533080101013, 0.02156640961766243, 0.999695360660553, 0.9645411968231201, 0.005901339929550886, 0.029506700113415718, 0.9996427297592163, 0.0007527430425398052, 0.0007203781860880554, 0.9991645216941833, 0.9995461106300354, 0.0010499433847144246, 0.9993538856506348, 0.0005423485417850316, 0.997921347618103, 0.0016270456835627556, 0.9998387694358826, 0.9992925524711609, 0.0006573516875505447, 0.0013147033751010895, 0.9985172152519226, 0.9987927675247192, 0.0012508362997323275, 0.7124689817428589, 0.22363123297691345, 0.06389463692903519, 0.9994685649871826, 0.004608211573213339, 0.9947977066040039, 0.0005760264466516674, 0.03518429026007652, 0.0005864048143848777, 0.964635968208313, 0.9991804361343384, 0.0008525430457666516, 0.9994620680809021, 0.007130985148251057, 0.9925439953804016, 0.47861841320991516, 0.5213650465011597, 0.00024014973314478993, 0.9983787536621094, 0.9991307258605957, 0.9998311996459961, 0.9985438585281372, 0.000665695930365473, 0.9996457695960999, 0.01617146097123623, 0.893162190914154, 0.09039431810379028, 0.998174250125885, 0.07260844856500626, 0.05884606018662453, 0.8689285516738892, 0.9996168613433838, 0.6280850172042847, 0.000656649237498641, 0.37133514881134033, 0.9990853667259216, 0.9901672005653381, 0.0009855678072199225, 0.008870110847055912, 0.9996187090873718, 0.9998704791069031, 0.0009353408822789788, 0.0009353408822789788, 0.998944103717804, 0.9998840093612671, 0.8121745586395264, 0.0008608103962615132, 0.18679584562778473, 0.002860341453924775, 0.03273501992225647, 0.9645707011222839, 0.9999033808708191, 0.9996041059494019, 0.9767000675201416, 0.02291378378868103, 0.0019260614644736052, 0.9981278777122498, 0.9995092153549194, 0.9990537762641907, 0.005875144619494677, 0.0036719655618071556, 0.9906963109970093, 0.9962024092674255, 0.0017467020079493523, 0.0017467020079493523, 0.9983338713645935, 0.002126657869666815, 0.005103978794068098, 0.9927238821983337, 0.9986677169799805, 0.0008884943672455847, 0.998012363910675, 0.18802951276302338, 0.21022523939609528, 0.6016627550125122, 0.005185074172914028, 0.49203622341156006, 0.5029521584510803, 0.0018663653172552586, 0.9985054731369019, 0.5805607438087463, 0.080917127430439, 0.3385949730873108, 0.9992635846138, 0.9995370507240295, 0.0008427799912169576, 0.999659538269043, 0.9971794486045837, 0.0023190220817923546, 0.0012887270422652364, 0.9987634420394897, 0.9991093873977661, 0.0008301432826556265, 0.9994925260543823, 0.6143405437469482, 0.006411904003471136, 0.37910380959510803, 0.06256932765245438, 0.10786204785108566, 0.8292770385742188, 0.3209436535835266, 0.678394615650177, 0.0004011795681435615, 0.9986591339111328, 0.9993519186973572, 0.000711789121851325, 0.9995976686477661, 0.0003550968540366739, 0.9988982677459717, 0.0005457315710373223, 0.0005457315710373223, 0.9992344975471497, 0.00035843768273480237, 0.03369314223527908, 0.9659895300865173, 0.001159834791906178, 0.0858277752995491, 0.9131765961647034, 0.006356926169246435, 0.9927399754524231, 0.0010594876948744059, 0.030815092846751213, 0.9687495231628418, 0.0016502278158441186, 0.9975627064704895, 0.9833773374557495, 0.01629907265305519, 0.0005832065944559872, 0.9990328550338745, 0.0005832065944559872, 0.9984256625175476, 0.00031988020054996014, 0.9993057250976562, 0.0006397604010999203, 0.003091915277764201, 0.0015459576388821006, 0.995315670967102, 0.016052355989813805, 0.9833813309669495, 0.9994829893112183, 0.06388549506664276, 0.2767465114593506, 0.6592439413070679, 0.9994003176689148, 0.12927863001823425, 0.2219935953617096, 0.6486783623695374, 0.00435299938544631, 0.9007226228713989, 0.0948953852057457, 0.0017812219448387623, 0.02374962531030178, 0.9749221205711365, 0.28673258423805237, 0.007848519831895828, 0.7053202986717224, 0.30968931317329407, 0.6891353726387024, 0.0011498365784063935], \"Term\": [\"afcon\", \"afcon\", \"africa\", \"africa\", \"africa\", \"anti\", \"anti\", \"anti\", \"arrest\", \"arrest\", \"arrested\", \"attack\", \"attack\", \"baby\", \"bank\", \"bank\", \"billion\", \"billion\", \"billion\", \"boda\", \"boost\", \"boost\", \"boss\", \"boy\", \"boy\", \"boy\", \"business\", \"business\", \"business\", \"call\", \"call\", \"call\", \"cancer\", \"cancer\", \"capital\", \"case\", \"case\", \"case\", \"cash\", \"cash\", \"cash\", \"cbk\", \"census\", \"census\", \"census\", \"champion\", \"champion\", \"charged\", \"child\", \"child\", \"child\", \"china\", \"china\", \"china\", \"chinese\", \"city\", \"city\", \"city\", \"coach\", \"coach\", \"coach\", \"company\", \"cost\", \"cost\", \"county\", \"county\", \"county\", \"court\", \"court\", \"court\", \"crash\", \"crash\", \"cup\", \"cup\", \"cut\", \"cut\", \"date\", \"date\", \"day\", \"day\", \"day\", \"dead\", \"deal\", \"death\", \"death\", \"debate\", \"debate\", \"debate\", \"debt\", \"debt\", \"die\", \"doctor\", \"doctor\", \"doctor\", \"eacc\", \"eacc\", \"economy\", \"election\", \"eye\", \"eye\", \"eye\", \"fake\", \"fake\", \"fake\", \"fall\", \"family\", \"family\", \"family\", \"farmer\", \"farmer\", \"father\", \"final\", \"final\", \"final\", \"finance\", \"fire\", \"fire\", \"fire\", \"firm\", \"firm\", \"fitness\", \"fitness\", \"five\", \"five\", \"food\", \"food\", \"food\", \"found\", \"free\", \"free\", \"fund\", \"fund\", \"get\", \"get\", \"get\", \"girl\", \"girl\", \"gor\", \"gor\", \"governor\", \"governor\", \"governor\", \"graft\", \"graft\", \"growth\", \"health\", \"health\", \"health\", \"hit\", \"hit\", \"hit\", \"home\", \"home\", \"home\", \"hotel\", \"housing\", \"husband\", \"iebc\", \"iebc\", \"investor\", \"jubilee\", \"jubilee\", \"kalonzo\", \"kalonzo\", \"kill\", \"kill\", \"killed\", \"killing\", \"killing\", \"knut\", \"kra\", \"kra\", \"kuria\", \"launch\", \"launch\", \"launch\", \"law\", \"law\", \"law\", \"leader\", \"leader\", \"leader\", \"league\", \"league\", \"league\", \"life\", \"life\", \"life\", \"loan\", \"local\", \"local\", \"local\", \"loss\", \"loss\", \"love\", \"love\", \"low\", \"low\", \"maize\", \"man\", \"man\", \"man\", \"market\", \"medium\", \"meet\", \"meet\", \"meet\", \"men\", \"men\", \"million\", \"million\", \"million\", \"missing\", \"mombasa\", \"mombasa\", \"mombasa\", \"money\", \"money\", \"money\", \"mother\", \"mother\", \"murder\", \"must\", \"must\", \"nairobi\", \"nairobi\", \"nairobi\", \"nse\", \"odm\", \"officer\", \"oil\", \"oil\", \"old\", \"one\", \"one\", \"one\", \"opposition\", \"order\", \"order\", \"order\", \"party\", \"pay\", \"pay\", \"pay\", \"personal\", \"plan\", \"plan\", \"plan\", \"play\", \"police\", \"political\", \"political\", \"political\", \"poll\", \"power\", \"power\", \"power\", \"president\", \"president\", \"president\", \"price\", \"profit\", \"project\", \"project\", \"raila\", \"raila\", \"rate\", \"referendum\", \"right\", \"right\", \"right\", \"rise\", \"rise\", \"rise\", \"ruling\", \"ruto\", \"ruto\", \"ruto\", \"safaricom\", \"safaricom\", \"salary\", \"say\", \"say\", \"say\", \"school\", \"school\", \"school\", \"senate\", \"senate\", \"set\", \"set\", \"set\", \"seven\", \"share\", \"share\", \"shot\", \"sign\", \"sign\", \"social\", \"social\", \"son\", \"sonko\", \"sonko\", \"state\", \"state\", \"state\", \"stop\", \"stop\", \"stop\", \"student\", \"student\", \"student\", \"supreme\", \"suspect\", \"suspect\", \"tax\", \"tax\", \"tea\", \"teacher\", \"teacher\", \"teacher\", \"team\", \"team\", \"team\", \"tell\", \"tell\", \"tell\", \"three\", \"three\", \"three\", \"title\", \"title\", \"told\", \"told\", \"trade\", \"trade\", \"trump\", \"trump\", \"trump\", \"tsc\", \"two\", \"two\", \"two\", \"uhuru\", \"uhuru\", \"uhuru\", \"use\", \"use\", \"vote\", \"want\", \"want\", \"want\", \"wife\", \"win\", \"win\", \"win\", \"woman\", \"woman\", \"woman\", \"work\", \"work\", \"work\", \"world\", \"world\", \"world\", \"year\", \"year\", \"year\"]}, \"R\": 30, \"lambda.step\": 0.01, \"plot.opts\": {\"xlab\": \"PC1\", \"ylab\": \"PC2\"}, \"topic.order\": [1, 2, 3]};\n",
       "\n",
       "function LDAvis_load_lib(url, callback){\n",
       "  var s = document.createElement('script');\n",
       "  s.src = url;\n",
       "  s.async = true;\n",
       "  s.onreadystatechange = s.onload = callback;\n",
       "  s.onerror = function(){console.warn(\"failed to load library \" + url);};\n",
       "  document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
       "}\n",
       "\n",
       "if(typeof(LDAvis) !== \"undefined\"){\n",
       "   // already loaded: just create the visualization\n",
       "   !function(LDAvis){\n",
       "       new LDAvis(\"#\" + \"ldavis_el1071620272854234727796538279\", ldavis_el1071620272854234727796538279_data);\n",
       "   }(LDAvis);\n",
       "}else if(typeof define === \"function\" && define.amd){\n",
       "   // require.js is available: use it to load d3/LDAvis\n",
       "   require.config({paths: {d3: \"https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.5/d3.min\"}});\n",
       "   require([\"d3\"], function(d3){\n",
       "      window.d3 = d3;\n",
       "      LDAvis_load_lib(\"https://cdn.rawgit.com/bmabey/pyLDAvis/files/ldavis.v1.0.0.js\", function(){\n",
       "        new LDAvis(\"#\" + \"ldavis_el1071620272854234727796538279\", ldavis_el1071620272854234727796538279_data);\n",
       "      });\n",
       "    });\n",
       "}else{\n",
       "    // require.js not available: dynamically load d3 & LDAvis\n",
       "    LDAvis_load_lib(\"https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.5/d3.min.js\", function(){\n",
       "         LDAvis_load_lib(\"https://cdn.rawgit.com/bmabey/pyLDAvis/files/ldavis.v1.0.0.js\", function(){\n",
       "                 new LDAvis(\"#\" + \"ldavis_el1071620272854234727796538279\", ldavis_el1071620272854234727796538279_data);\n",
       "            })\n",
       "         });\n",
       "}\n",
       "</script>"
      ],
      "text/plain": [
       "PreparedData(topic_coordinates=              x         y  topics  cluster       Freq\n",
       "topic                                                \n",
       "0      0.233834 -0.170223       1        1  33.204533\n",
       "1     -0.281962 -0.095766       2        1  32.111397\n",
       "2      0.048128  0.265988       3        1  34.684071, topic_info=     Category         Freq       Term        Total  loglift  logprob\n",
       "term                                                                \n",
       "641   Default  7115.000000      uhuru  7115.000000  30.0000  30.0000\n",
       "431   Default  5531.000000        man  5531.000000  29.0000  29.0000\n",
       "139   Default  4565.000000     police  4565.000000  28.0000  28.0000\n",
       "81    Default  4645.000000     market  4645.000000  27.0000  27.0000\n",
       "12    Default  4914.000000        get  4914.000000  26.0000  26.0000\n",
       "203   Default  4672.000000      raila  4672.000000  25.0000  25.0000\n",
       "547   Default  4702.000000       ruto  4702.000000  24.0000  24.0000\n",
       "613   Default  5743.000000      woman  5743.000000  23.0000  23.0000\n",
       "226   Default  4120.000000       firm  4120.000000  22.0000  22.0000\n",
       "842   Default  4038.000000       bank  4038.000000  21.0000  21.0000\n",
       "307   Default  4584.000000      court  4584.000000  20.0000  20.0000\n",
       "0     Default  3714.000000    company  3714.000000  19.0000  19.0000\n",
       "219   Default  3376.000000       deal  3376.000000  18.0000  18.0000\n",
       "17    Default  3411.000000     africa  3411.000000  17.0000  17.0000\n",
       "504   Default  3126.000000        two  3126.000000  16.0000  16.0000\n",
       "2459  Default  3692.000000   governor  3692.000000  15.0000  15.0000\n",
       "143   Default  3451.000000     leader  3451.000000  14.0000  14.0000\n",
       "300   Default  3043.000000       plan  3043.000000  13.0000  13.0000\n",
       "111   Default  2856.000000     farmer  2856.000000  12.0000  12.0000\n",
       "732   Default  2746.000000      death  2746.000000  11.0000  11.0000\n",
       "897   Default  2816.000000        tax  2816.000000  10.0000  10.0000\n",
       "1871  Default  3146.000000  president  3146.000000   9.0000   9.0000\n",
       "698   Default  2689.000000       life  2689.000000   8.0000   8.0000\n",
       "972   Default  5218.000000       year  5218.000000   7.0000   7.0000\n",
       "293   Default  2253.000000     killed  2253.000000   6.0000   6.0000\n",
       "138   Default  2789.000000       team  2789.000000   5.0000   5.0000\n",
       "725   Default  2203.000000     attack  2203.000000   4.0000   4.0000\n",
       "2648  Default  2145.000000   arrested  2145.000000   3.0000   3.0000\n",
       "3577  Default  2295.000000    jubilee  2295.000000   2.0000   2.0000\n",
       "49    Default  2236.000000   business  2236.000000   1.0000   1.0000\n",
       "...       ...          ...        ...          ...      ...      ...\n",
       "1746   Topic3   839.123352       anti   841.538025   1.0560  -6.5247\n",
       "202    Topic3   741.089172    kalonzo   742.343872   1.0572  -6.6490\n",
       "143    Topic3  3380.374512     leader  3451.226562   1.0381  -5.1313\n",
       "307    Topic3  4442.209961      court  4584.145508   1.0274  -4.8582\n",
       "15     Topic3  1349.024536      right  1361.668579   1.0496  -6.0499\n",
       "2459   Topic3  3570.615234   governor  3692.009033   1.0255  -5.0766\n",
       "1871   Topic3  3034.852051  president  3146.477539   1.0228  -5.2392\n",
       "1029   Topic3  1444.018066      final  1465.200317   1.0443  -5.9819\n",
       "138    Topic3  2694.731689       team  2789.885254   1.0242  -5.3580\n",
       "470    Topic3  1408.987671        use  1432.811401   1.0421  -6.0064\n",
       "888    Topic3  1641.687256       work  1684.237061   1.0333  -5.8536\n",
       "2260   Topic3  1194.732300       fake  1209.951172   1.0462  -6.1714\n",
       "1371   Topic3  1460.393555       free  1497.685181   1.0337  -5.9706\n",
       "835    Topic3  1644.590210      money  1705.306519   1.0226  -5.8518\n",
       "944    Topic3  2361.507568       tell  2586.575195   0.9679  -5.4900\n",
       "1235   Topic3  1978.676880     census  2125.538330   0.9873  -5.6669\n",
       "770    Topic3  1508.915771      title  1557.678223   1.0271  -5.9379\n",
       "890    Topic3  1833.859619        law  2099.139893   0.9238  -5.7429\n",
       "1203   Topic3  1830.856445      order  2107.192871   0.9183  -5.7445\n",
       "167    Topic3  3794.764648        say  6307.520508   0.5508  -5.0157\n",
       "48     Topic3  2696.061279      world  3822.376953   0.7098  -5.3575\n",
       "208    Topic3  1972.087158       call  2513.278320   0.8164  -5.6702\n",
       "512    Topic3  2425.330322       want  3678.456299   0.6424  -5.4633\n",
       "468    Topic3  1776.066650       stop  2141.624512   0.8717  -5.7749\n",
       "3105   Topic3  2660.949219     county  6039.285156   0.2393  -5.3706\n",
       "398    Topic3  1947.311646       case  2766.386230   0.7078  -5.6829\n",
       "159    Topic3  1987.414429        win  3063.151367   0.6263  -5.6625\n",
       "879    Topic3  1895.365845     health  3040.203613   0.5864  -5.7099\n",
       "943    Topic3  1892.344604      state  4990.717285   0.0891  -5.7115\n",
       "47     Topic3  1842.577026     school  3664.364258   0.3714  -5.7382\n",
       "\n",
       "[212 rows x 6 columns], token_table=      Topic      Freq      Term\n",
       "term                           \n",
       "9994      1  0.001246     afcon\n",
       "9994      3  0.998228     afcon\n",
       "17        1  0.995054    africa\n",
       "17        2  0.000586    africa\n",
       "17        3  0.004396    africa\n",
       "1746      1  0.002377      anti\n",
       "1746      2  0.001188      anti\n",
       "1746      3  0.996984      anti\n",
       "353       2  0.998269    arrest\n",
       "353       3  0.000908    arrest\n",
       "2648      2  0.999724  arrested\n",
       "725       1  0.000908    attack\n",
       "725       2  0.998567    attack\n",
       "3880      2  0.999548      baby\n",
       "842       1  0.999723      bank\n",
       "842       3  0.000248      bank\n",
       "218       1  0.997892   billion\n",
       "218       2  0.000520   billion\n",
       "218       3  0.001560   billion\n",
       "5998      2  0.998429      boda\n",
       "66        1  0.992400     boost\n",
       "66        3  0.007265     boost\n",
       "4264      3  0.997983      boss\n",
       "248       1  0.000788       boy\n",
       "248       2  0.997076       boy\n",
       "248       3  0.002363       boy\n",
       "49        1  0.993775  business\n",
       "49        2  0.002682  business\n",
       "49        3  0.003129  business\n",
       "208       1  0.181834      call\n",
       "...     ...       ...       ...\n",
       "4548      3  0.000583     trump\n",
       "1912      3  0.998426       tsc\n",
       "504       1  0.000320       two\n",
       "504       2  0.999306       two\n",
       "504       3  0.000640       two\n",
       "641       1  0.003092     uhuru\n",
       "641       2  0.001546     uhuru\n",
       "641       3  0.995316     uhuru\n",
       "470       1  0.016052       use\n",
       "470       3  0.983381       use\n",
       "494       3  0.999483      vote\n",
       "512       1  0.063885      want\n",
       "512       2  0.276747      want\n",
       "512       3  0.659244      want\n",
       "1523      2  0.999400      wife\n",
       "159       1  0.129279       win\n",
       "159       2  0.221994       win\n",
       "159       3  0.648678       win\n",
       "613       1  0.004353     woman\n",
       "613       2  0.900723     woman\n",
       "613       3  0.094895     woman\n",
       "888       1  0.001781      work\n",
       "888       2  0.023750      work\n",
       "888       3  0.974922      work\n",
       "48        1  0.286733     world\n",
       "48        2  0.007849     world\n",
       "48        3  0.705320     world\n",
       "972       1  0.309689      year\n",
       "972       2  0.689135      year\n",
       "972       3  0.001150      year\n",
       "\n",
       "[368 rows x 3 columns], R=30, lambda_step=0.01, plot_opts={'xlab': 'PC1', 'ylab': 'PC2'}, topic_order=[1, 2, 3])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pyLDAvis.enable_notebook()\n",
    "vis = pyLDAvis.gensim.prepare(lda_model, bow, dictionary,  sort_topics = False)\n",
    "vis"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}