lda_model_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/lwgray/Envs/pocket/local/lib/python2.7/site-packages/matplotlib/font_manager.py:273: UserWarning: Matplotlib is building the font cache using fc-list. This may take a moment.\n",
      "  warnings.warn('Matplotlib is building the font cache using fc-list. This may take a moment.')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(0, u'0.437*python + 0.074*learning + 0.068*new + 0.056*web + 0.050*learn + 0.048*tutorial + 0.045*way + 0.036*free + 0.035*use + 0.030*deep'), (1, u'0.147*deep_learning + 0.099*ai + 0.073*building + 0.072*regression + 0.066*tensorflow + 0.066*make + 0.066*spark + 0.061*business + 0.061*app + 0.061*getting'), (2, u'0.232*data_science + 0.232*machine_learning + 0.091*using + 0.061*part + 0.043*introduction + 0.040*guide + 0.038*programming + 0.038*code + 0.034*models + 0.030*algorithm'), (3, u'0.254*data + 0.081*big_data + 0.077*medium + 0.077*build + 0.063*analysis + 0.060*product + 0.049*software + 0.046*data_scientists + 0.043*developer + 0.042*ways')]\n"
     ]
    }
   ],
   "source": [
    "# %load POCKET_TOPIC_MODELING.py\n",
    "import gensim\n",
    "import pyLDAvis\n",
    "import pyLDAvis.gensim\n",
    "import numpy as np\n",
    "from wordcloud import WordCloud\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "dictionary = gensim.corpora.Dictionary.load('larry.dict')\n",
    "corpus = gensim.corpora.MmCorpus('larry.mm')\n",
    "lda = gensim.models.ldamodel.LdaModel.load('larry.lda')\n",
    "\n",
    "print lda.show_topics()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<link rel=\"stylesheet\" type=\"text/css\" href=\"https://cdn.rawgit.com/bmabey/pyLDAvis/files/ldavis.v1.0.0.css\">\n",
       "\n",
       "\n",
       "<div id=\"ldavis_el1817030282017729196948528\"></div>\n",
       "<script type=\"text/javascript\">\n",
       "\n",
       "var ldavis_el1817030282017729196948528_data = {\"plot.opts\": {\"xlab\": \"PC1\", \"ylab\": \"PC2\"}, \"topic.order\": [3, 1, 4, 2], \"token.table\": {\"Topic\": [4, 1, 3, 4, 2, 3, 3, 3, 1, 4, 4, 1, 3, 1, 4, 3, 2, 4, 3, 4, 2, 4, 4, 1, 1, 1, 1, 2, 2, 2, 1, 4, 3, 1, 1, 3, 2, 1, 1, 3, 1, 2, 4, 2, 3, 4, 1, 4, 3, 3, 2, 2, 3, 1, 2, 3, 2], \"Freq\": [0.97531015200427518, 0.95770095032493918, 0.94471589028889902, 0.90730161041607493, 0.9457020071143859, 0.96777034620366309, 0.9679108636947289, 0.96408247476713227, 0.50921294466405609, 0.47004271815143633, 0.90730361720832342, 0.9717336763089548, 0.98465960376164685, 0.99680763024616426, 0.90729567634077135, 0.98037001291240322, 0.94566478352435124, 0.97388513978183366, 0.97456846496319316, 0.89051722745835282, 0.96005114190039331, 0.90730222097081414, 0.89051288202119538, 0.97434272557214463, 0.93341397927493619, 0.97674283730029621, 0.94102833914883344, 0.98199977119774884, 0.95958450004686147, 0.94566326147297974, 0.9968083269998862, 0.92148383044165127, 0.96407371796186103, 0.96554417026966721, 0.93342369250735424, 0.95972015001646882, 0.95295595151934609, 0.93341736489838656, 0.99099987202674045, 0.9385331309382744, 0.97172136204217641, 0.99756255489226009, 0.93365069092733888, 0.94570672396266664, 0.91503319196765143, 0.92145757727351085, 0.94099274855388826, 0.92149627295223902, 0.96776213997289107, 0.95971920437995262, 0.97865381233835491, 0.66011105329411601, 0.33005552664705801, 0.97829564669363278, 0.97487208210270404, 0.97454361189317096, 0.98772076007377996], \"Term\": [\"ai\", \"algorithm\", \"analysis\", \"app\", \"applications\", \"artificial_intelligence\", \"big_data\", \"build\", \"building\", \"building\", \"business\", \"code\", \"data\", \"data_science\", \"data_scientist\", \"data_scientists\", \"deep\", \"deep_learning\", \"developer\", \"flask\", \"free\", \"getting\", \"good\", \"guide\", \"infrastructure\", \"introduction\", \"know\", \"learn\", \"learning\", \"list\", \"machine_learning\", \"make\", \"medium\", \"models\", \"need\", \"neural_networks\", \"new\", \"one\", \"part\", \"product\", \"programming\", \"python\", \"regression\", \"simple\", \"software\", \"spark\", \"technical\", \"tensorflow\", \"tools\", \"top\", \"tutorial\", \"use\", \"use\", \"using\", \"way\", \"ways\", \"web\"]}, \"mdsDat\": {\"y\": [-0.073979853903078319, -0.27658886559365625, 0.31388822409650347, 0.036680495400230974], \"cluster\": [1, 1, 1, 1], \"Freq\": [37.589101038494583, 26.62497078951322, 22.072943081183798, 13.712985090808393], \"topics\": [1, 2, 3, 4], \"x\": [0.31575281961538554, -0.25858143009613982, -0.16617257489754528, 0.10900118537829945]}, \"R\": 30, \"lambda.step\": 0.01, \"tinfo\": {\"Category\": [\"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\"], \"Term\": [\"python\", \"data\", \"data_science\", \"machine_learning\", \"deep_learning\", \"using\", \"ai\", \"big_data\", \"medium\", \"build\", \"learning\", \"building\", \"new\", \"analysis\", \"regression\", \"part\", \"product\", \"tensorflow\", \"make\", \"spark\", \"web\", \"business\", \"app\", \"getting\", \"data_scientist\", \"learn\", \"software\", \"flask\", \"good\", \"tutorial\", \"data_science\", \"machine_learning\", \"using\", \"part\", \"introduction\", \"guide\", \"programming\", \"code\", \"models\", \"algorithm\", \"technical\", \"know\", \"infrastructure\", \"need\", \"one\", \"building\", \"good\", \"flask\", \"neural_networks\", \"top\", \"list\", \"deep\", \"applications\", \"simple\", \"data_scientist\", \"business\", \"getting\", \"app\", \"tools\", \"artificial_intelligence\", \"spark\", \"free\", \"make\", \"learn\", \"new\", \"medium\", \"use\", \"python\", \"build\", \"data\", \"python\", \"learning\", \"new\", \"web\", \"learn\", \"tutorial\", \"way\", \"free\", \"deep\", \"list\", \"applications\", \"simple\", \"use\", \"flask\", \"good\", \"neural_networks\", \"top\", \"one\", \"need\", \"infrastructure\", \"app\", \"getting\", \"data_scientist\", \"business\", \"tools\", \"artificial_intelligence\", \"technical\", \"know\", \"make\", \"tensorflow\", \"ways\", \"algorithm\", \"models\", \"introduction\", \"programming\", \"regression\", \"build\", \"building\", \"analysis\", \"using\", \"code\", \"data\", \"machine_learning\", \"data_science\", \"data\", \"big_data\", \"medium\", \"build\", \"analysis\", \"product\", \"software\", \"data_scientists\", \"developer\", \"ways\", \"tools\", \"artificial_intelligence\", \"top\", \"neural_networks\", \"use\", \"good\", \"flask\", \"simple\", \"deep\", \"list\", \"applications\", \"need\", \"one\", \"infrastructure\", \"data_scientist\", \"getting\", \"app\", \"business\", \"know\", \"technical\", \"spark\", \"building\", \"code\", \"learning\", \"tutorial\", \"programming\", \"new\", \"using\", \"web\", \"part\", \"deep_learning\", \"ai\", \"regression\", \"tensorflow\", \"make\", \"spark\", \"business\", \"app\", \"getting\", \"data_scientist\", \"flask\", \"good\", \"building\", \"simple\", \"applications\", \"neural_networks\", \"top\", \"list\", \"deep\", \"need\", \"infrastructure\", \"one\", \"artificial_intelligence\", \"tools\", \"know\", \"technical\", \"developer\", \"ways\", \"free\", \"data_scientists\", \"software\", \"algorithm\", \"models\", \"build\", \"introduction\", \"code\", \"web\", \"using\", \"machine_learning\", \"part\", \"data\"], \"loglift\": [30.0, 29.0, 28.0, 27.0, 26.0, 25.0, 24.0, 23.0, 22.0, 21.0, 20.0, 19.0, 18.0, 17.0, 16.0, 15.0, 14.0, 13.0, 12.0, 11.0, 10.0, 9.0, 8.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, 0.9718, 0.9718, 0.96150000000000002, 0.9536, 0.9425, 0.94120000000000004, 0.93879999999999997, 0.93869999999999998, 0.93379999999999996, 0.92759999999999998, 0.91559999999999997, 0.91449999999999998, 0.90969999999999995, 0.90939999999999999, 0.90910000000000002, 0.27889999999999998, -2.7162999999999999, -2.7376, -2.7480000000000002, -2.7526000000000002, -2.7576999999999998, -2.7631999999999999, -2.7709999999999999, -2.7753000000000001, -2.8104, -2.8108, -2.8220000000000001, -2.8220999999999998, -2.8469000000000002, -2.8473999999999999, -2.8725999999999998, -2.9333, -2.8952, -3.2555000000000001, -3.5432000000000001, -3.4870000000000001, -3.3096000000000001, -5.3785999999999996, -3.4891999999999999, -4.6577000000000002, 1.3183, 1.2939000000000001, 1.2915000000000001, 1.2850999999999999, 1.2809999999999999, 1.2782, 1.2755000000000001, 1.2639, 1.2525999999999999, 1.2524999999999999, 1.2521, 1.252, 0.87829999999999997, -2.3618999999999999, -2.4007999999999998, -2.4039000000000001, -2.4091, -2.4390999999999998, -2.4632999999999998, -2.464, -2.4813000000000001, -2.4874000000000001, -2.4958999999999998, -2.4973000000000001, -2.5177, -2.5205000000000002, -2.5421999999999998, -2.5442999999999998, -2.5667, -2.5720000000000001, -2.5754999999999999, -2.7408000000000001, -2.8660999999999999, -3.0878000000000001, -2.9912999999999998, -2.6395, -3.1562999999999999, -3.3136999999999999, -2.9700000000000002, -3.8313000000000001, -3.0005999999999999, -4.3292999999999999, -4.7656999999999998, -4.7657999999999996, 1.5004, 1.4785999999999999, 1.4770000000000001, 1.4765999999999999, 1.4697, 1.4678, 1.4584999999999999, 1.4548000000000001, 1.45, 1.4499, 1.4451000000000001, 1.4450000000000001, 1.4383999999999999, 1.4379, 0.41049999999999998, -2.2151000000000001, -2.2391000000000001, -2.2635000000000001, -2.2677, -2.2685, -2.2713000000000001, -2.2869000000000002, -2.2881999999999998, -2.2976000000000001, -2.3138000000000001, -2.3138000000000001, -2.3220999999999998, -2.3231000000000002, -2.3357000000000001, -2.3847999999999998, -2.3948, -3.1173000000000002, -2.8197999999999999, -3.1103999999999998, -2.7008999999999999, -2.8256999999999999, -3.0457999999999998, -3.665, -2.8673000000000002, -3.2894000000000001, 1.9579, 1.9441999999999999, 1.9282999999999999, 1.9239999999999999, 1.9237, 1.9231, 1.9184000000000001, 1.9182999999999999, 1.9181999999999999, 1.9180999999999999, 1.9111, 1.9109, 1.2605999999999999, -1.8106, -1.8111999999999999, -1.8220000000000001, -1.8303, -1.8444, -1.8449, -1.8476999999999999, -1.8496999999999999, -1.8574999999999999, -1.8999999999999999, -1.9103000000000001, -1.9399999999999999, -1.9453, -1.9682999999999999, -1.9925999999999999, -2.0023, -2.0682, -2.1284000000000001, -2.1536, -2.274, -2.5369999999999999, -2.4828999999999999, -2.3875999999999999, -2.4291999999999998, -3.2315999999999998, -4.1590999999999996, -2.8506, -3.7263000000000002], \"Freq\": [144.0, 70.0, 108.0, 108.0, 25.0, 42.0, 17.0, 22.0, 21.0, 21.0, 25.0, 25.0, 23.0, 17.0, 12.0, 29.0, 17.0, 11.0, 11.0, 11.0, 19.0, 11.0, 11.0, 11.0, 11.0, 17.0, 14.0, 10.0, 10.0, 16.0, 107.62891650854154, 107.62771052644204, 42.208835686917574, 28.545809238130047, 19.752416498074865, 18.786362874261666, 17.803848529589278, 17.801112423020914, 15.846801789432691, 13.893779087031966, 10.978030191091891, 10.964985708813279, 10.001897919166664, 9.9986149048512818, 9.9952999357626631, 12.683690980189409, 0.2511766573519138, 0.24587660091838512, 0.25087450398494199, 0.24974340817506499, 0.25216195148580145, 0.25077134037480675, 0.24881234075792172, 0.24773982689218396, 0.24933084942707009, 0.24922560610838701, 0.24643905353564183, 0.2464381250895368, 0.24791165388867653, 0.24778133126089805, 0.25376740497657102, 0.25006308898892093, 0.24808156366750386, 0.25093435725782559, 0.25098086693885346, 0.25049322550136466, 0.24962574651702901, 0.25036928236668948, 0.2499378187686308, 0.24993552871750799, 143.63011646425704, 24.285321948083133, 22.362512477371173, 18.514981048278781, 16.593582412072028, 15.627689182008845, 14.66867481106838, 11.778242949735288, 9.8529430602934394, 9.8516339486710596, 9.8474932420076495, 9.8465045597712493, 11.649697795431422, 0.25359593570129474, 0.24390616501875095, 0.25068477478730128, 0.24938369829610554, 0.24883499293603134, 0.24289266235392704, 0.24272896074051237, 0.24542891422958774, 0.24393226435479645, 0.24186584385872828, 0.24153586360654675, 0.24405402708317214, 0.24338169107879906, 0.24492585237172057, 0.24440665367011963, 0.24405160654228897, 0.2427575725617481, 0.24954456602053998, 0.25112391381377019, 0.25114302312128062, 0.24861324862244219, 0.24769044049944539, 0.24431718558158647, 0.2469566144882607, 0.24729447003178184, 0.24579602461523975, 0.24783746882444171, 0.24540354733840827, 0.24584622763805247, 0.24569332008408651, 0.24565790727109846, 69.347788549366868, 22.007740476104992, 21.058621881305175, 21.049202394732475, 17.269650100799247, 16.329459970268211, 13.482928213614231, 12.537833353908056, 11.586997762305543, 11.585463164849143, 10.643716440425067, 10.642040745901076, 9.6914424078748311, 9.6870935743813522, 6.0493150723078069, 0.24348317287501409, 0.23770600469081229, 0.24269890573668906, 0.24168953456551451, 0.24151496645394679, 0.24082141353814732, 0.24021937398711338, 0.23990310072295506, 0.23765544747251394, 0.24056701517003312, 0.24056307190967363, 0.2385878468434855, 0.23833826278941991, 0.24960545315724719, 0.23767276822672054, 0.24028285093768176, 0.24950101640782379, 0.2437571344580913, 0.24612324974181646, 0.24231188285911961, 0.24233795764688518, 0.24234134610668792, 0.24262354242212608, 0.24138428732197806, 0.2407740595647575, 24.939583560153348, 16.702902348036208, 12.122709120919914, 11.209779884496374, 11.206566084987513, 11.200957282247405, 10.292569100657509, 10.291238325000268, 10.290751404487917, 10.290001588973071, 9.3693075798924887, 9.367969442594049, 12.349108050212093, 0.2371593830026251, 0.23702841921975132, 0.2310512656235289, 0.22914487123901481, 0.22927779238377377, 0.2291677039167386, 0.23152158707043352, 0.23107768439972387, 0.22928312363898351, 0.23312920853458499, 0.23074723727600405, 0.23034196550725713, 0.22915308822837607, 0.23586515339733566, 0.23022552233544596, 0.23143053394219401, 0.22987138578888747, 0.2318822288268142, 0.23266588704412328, 0.23382681624986015, 0.23627195230861783, 0.2344593060329154, 0.23332079631342073, 0.23242265592234612, 0.23251041142315207, 0.23210398618631148, 0.23198282462528402, 0.23140762208398491], \"Total\": [144.0, 70.0, 108.0, 108.0, 25.0, 42.0, 17.0, 22.0, 21.0, 21.0, 25.0, 25.0, 23.0, 17.0, 12.0, 29.0, 17.0, 11.0, 11.0, 11.0, 19.0, 11.0, 11.0, 11.0, 11.0, 17.0, 14.0, 10.0, 10.0, 16.0, 108.34588011061786, 108.34580437851051, 42.93180710958729, 29.263374111936802, 20.476218750965941, 19.500325194959498, 18.523828643811097, 18.523593901130834, 16.570966396629327, 14.61834197329545, 11.689781899918708, 11.689339781147904, 10.713360011779415, 10.713248528262756, 10.713321153060633, 25.529594516841108, 10.106535437839728, 10.10648612120298, 10.419704118777124, 10.419714385585017, 10.574588658994582, 10.574571639150498, 10.574155415523471, 10.574102675402747, 11.021765297428903, 11.021668833161863, 11.021685794288029, 11.021693211162878, 11.366429358672919, 11.366332976775357, 11.937608709613915, 12.499334125311647, 11.937268605926477, 17.311612994843202, 23.086061811067218, 21.782566632348299, 18.178759377103379, 144.35184970986853, 21.782368780297983, 70.074977927806401, 144.35184970986853, 25.010824996472905, 23.086061811067218, 19.236205988604262, 17.311612994843202, 16.348988578269839, 15.386634077823279, 12.499334125311647, 10.574571639150498, 10.574588658994582, 10.574155415523471, 10.574102675402747, 18.178759377103379, 10.10648612120298, 10.106535437839728, 10.419704118777124, 10.419714385585017, 10.713321153060633, 10.713248528262756, 10.713360011779415, 11.021693211162878, 11.021685794288029, 11.021765297428903, 11.021668833161863, 11.366429358672919, 11.366332976775357, 11.689781899918708, 11.689339781147904, 11.937268605926477, 11.937107422864344, 12.313456117872983, 14.61834197329545, 16.570966396629327, 20.476218750965941, 18.523828643811097, 12.852772580375991, 21.782368780297983, 25.529594516841108, 17.994828048040254, 42.93180710958729, 18.523593901130834, 70.074977927806401, 108.34580437851051, 108.34588011061786, 70.074977927806401, 22.729365714546436, 21.782566632348299, 21.782368780297983, 17.994828048040254, 17.047879795148425, 14.207134904085075, 13.260299508122102, 12.313142104852744, 12.313456117872983, 11.366429358672919, 11.366332976775357, 10.419714385585017, 10.419704118777124, 18.178759377103379, 10.106535437839728, 10.10648612120298, 10.574102675402747, 10.574571639150498, 10.574588658994582, 10.574155415523471, 10.713248528262756, 10.713321153060633, 10.713360011779415, 11.021765297428903, 11.021685794288029, 11.021693211162878, 11.021668833161863, 11.689339781147904, 11.689781899918708, 11.937608709613915, 25.529594516841108, 18.523593901130834, 25.010824996472905, 16.348988578269839, 18.523828643811097, 23.086061811067218, 42.93180710958729, 19.236205988604262, 29.263374111936802, 25.670378342152762, 17.430352760160218, 12.852772580375991, 11.937107422864344, 11.937268605926477, 11.937608709613915, 11.021668833161863, 11.021693211162878, 11.021685794288029, 11.021765297428903, 10.10648612120298, 10.106535437839728, 25.529594516841108, 10.574102675402747, 10.574155415523471, 10.419704118777124, 10.419714385585017, 10.574588658994582, 10.574571639150498, 10.713248528262756, 10.713360011779415, 10.713321153060633, 11.366332976775357, 11.366429358672919, 11.689339781147904, 11.689781899918708, 12.313142104852744, 12.313456117872983, 12.499334125311647, 13.260299508122102, 14.207134904085075, 14.61834197329545, 16.570966396629327, 21.782368780297983, 20.476218750965941, 18.523593901130834, 19.236205988604262, 42.93180710958729, 108.34580437851051, 29.263374111936802, 70.074977927806401], \"logprob\": [30.0, 29.0, 28.0, 27.0, 26.0, 25.0, 24.0, 23.0, 22.0, 21.0, 20.0, 19.0, 18.0, 17.0, 16.0, 15.0, 14.0, 13.0, 12.0, 11.0, 10.0, 9.0, 8.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, -1.4617, -1.4617, -2.3976999999999999, -2.7888999999999999, -3.1570999999999998, -3.2071999999999998, -3.2610000000000001, -3.2610999999999999, -3.3774000000000002, -3.5089000000000001, -3.7444999999999999, -3.7456999999999998, -3.8376000000000001, -3.8378999999999999, -3.8382999999999998, -3.6000999999999999, -7.5220000000000002, -7.5433000000000003, -7.5232000000000001, -7.5277000000000003, -7.5180999999999996, -7.5236000000000001, -7.5313999999999997, -7.5357000000000003, -7.5293000000000001, -7.5297999999999998, -7.5410000000000004, -7.5410000000000004, -7.5350999999999999, -7.5355999999999996, -7.5117000000000003, -7.5263999999999998, -7.5343999999999998, -7.5228999999999999, -7.5227000000000004, -7.5247000000000002, -7.5282, -7.5251999999999999, -7.5269000000000004, -7.5269000000000004, -0.82830000000000004, -2.6055999999999999, -2.6880999999999999, -2.8769, -2.9864999999999999, -3.0465, -3.1097999999999999, -3.3292999999999999, -3.5076999999999998, -3.5078999999999998, -3.5083000000000002, -3.5084, -3.3401999999999998, -7.1675000000000004, -7.2065000000000001, -7.1791, -7.1843000000000004, -7.1864999999999997, -7.2106000000000003, -7.2112999999999996, -7.2003000000000004, -7.2064000000000004, -7.2149000000000001, -7.2161999999999997, -7.2058999999999997, -7.2085999999999997, -7.2023000000000001, -7.2043999999999997, -7.2058999999999997, -7.2111999999999998, -7.1836000000000002, -7.1772999999999998, -7.1772, -7.1874000000000002, -7.1910999999999996, -7.2047999999999996, -7.194, -7.1927000000000003, -7.1988000000000003, -7.1905000000000001, -7.2004000000000001, -7.1985999999999999, -7.1992000000000003, -7.1993, -1.3689, -2.5165999999999999, -2.5607000000000002, -2.5611000000000002, -2.7591000000000001, -2.8149999999999999, -3.0066000000000002, -3.0792999999999999, -3.1581000000000001, -3.1583000000000001, -3.2429999999999999, -3.2431999999999999, -3.3368000000000002, -3.3372000000000002, -3.8081, -7.0206999999999997, -7.0446999999999997, -7.0239000000000003, -7.0281000000000002, -7.0288000000000004, -7.0316999999999998, -7.0342000000000002, -7.0354999999999999, -7.0449000000000002, -7.0327999999999999, -7.0327999999999999, -7.0410000000000004, -7.0420999999999996, -6.9958999999999998, -7.0449000000000002, -7.0339, -6.9962999999999997, -7.0195999999999996, -7.0099, -7.0255000000000001, -7.0254000000000003, -7.0254000000000003, -7.0243000000000002, -7.0293999999999999, -7.0319000000000003, -1.9155, -2.3163999999999998, -2.6368999999999998, -2.7151999999999998, -2.7155, -2.7160000000000002, -2.8006000000000002, -2.8007, -2.8008000000000002, -2.8008000000000002, -2.8946000000000001, -2.8946999999999998, -2.6183999999999998, -6.5709999999999997, -6.5716000000000001, -6.5971000000000002, -6.6054000000000004, -6.6048, -6.6052999999999997, -6.5951000000000004, -6.5970000000000004, -6.6048, -6.5881999999999996, -6.5983999999999998, -6.6002000000000001, -6.6054000000000004, -6.5765000000000002, -6.6006999999999998, -6.5955000000000004, -6.6021999999999998, -6.5934999999999997, -6.5902000000000003, -6.5852000000000004, -6.5747999999999998, -6.5824999999999996, -6.5872999999999999, -6.5911999999999997, -6.5907999999999998, -6.5926, -6.5930999999999997, -6.5956000000000001]}};\n",
       "\n",
       "function LDAvis_load_lib(url, callback){\n",
       "  var s = document.createElement('script');\n",
       "  s.src = url;\n",
       "  s.async = true;\n",
       "  s.onreadystatechange = s.onload = callback;\n",
       "  s.onerror = function(){console.warn(\"failed to load library \" + url);};\n",
       "  document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
       "}\n",
       "\n",
       "if(typeof(LDAvis) !== \"undefined\"){\n",
       "   // already loaded: just create the visualization\n",
       "   !function(LDAvis){\n",
       "       new LDAvis(\"#\" + \"ldavis_el1817030282017729196948528\", ldavis_el1817030282017729196948528_data);\n",
       "   }(LDAvis);\n",
       "}else if(typeof define === \"function\" && define.amd){\n",
       "   // require.js is available: use it to load d3/LDAvis\n",
       "   require.config({paths: {d3: \"https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.5/d3.min\"}});\n",
       "   require([\"d3\"], function(d3){\n",
       "      window.d3 = d3;\n",
       "      LDAvis_load_lib(\"https://cdn.rawgit.com/bmabey/pyLDAvis/files/ldavis.v1.0.0.js\", function(){\n",
       "        new LDAvis(\"#\" + \"ldavis_el1817030282017729196948528\", ldavis_el1817030282017729196948528_data);\n",
       "      });\n",
       "    });\n",
       "}else{\n",
       "    // require.js not available: dynamically load d3 & LDAvis\n",
       "    LDAvis_load_lib(\"https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.5/d3.min.js\", function(){\n",
       "         LDAvis_load_lib(\"https://cdn.rawgit.com/bmabey/pyLDAvis/files/ldavis.v1.0.0.js\", function(){\n",
       "                 new LDAvis(\"#\" + \"ldavis_el1817030282017729196948528\", ldavis_el1817030282017729196948528_data);\n",
       "            })\n",
       "         });\n",
       "}\n",
       "</script>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "topics = pyLDAvis.gensim.prepare(lda, corpus, dictionary)\n",
    "pyLDAvis.display(topics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0, [(u'python', 0.43680682455640296), (u'learning', 0.073856337546812159), (u'new', 0.068008703918124816), (u'web', 0.056307620417708391), (u'learn', 0.050464277408254432), (u'tutorial', 0.047526810217733709), (u'way', 0.044610262961582818), (u'free', 0.035819903432355853), (u'use', 0.035428972880701265), (u'deep', 0.029964695961051113), (u'list', 0.029960714700680114), (u'applications', 0.029948122014873065), (u'simple', 0.029945115241928725), (u'flask', 0.00077123404283849109), (u'models', 0.00076377426364060235), (u'algorithm', 0.00076371614855904018), (u'neural_networks', 0.00076238064227099496), (u'ways', 0.00075891304798783978), (u'top', 0.00075842381828021565), (u'one', 0.00075675509968657216), (u'introduction', 0.00075608073255614922), (u'using', 0.00075372143689823661), (u'programming', 0.00075327429546757799), (u'building', 0.00075207007307428022), (u'build', 0.00075104258934902005), (u'data', 0.00074766568925307573), (u'analysis', 0.0007475130121995595), (u'machine_learning', 0.00074720066795571304), (u'data_science', 0.00074709297077652317), (u'app', 0.00074639655886953493), (u'code', 0.00074631941327181679), (u'guide', 0.00074537430504868405), (u'technical', 0.00074486665094980497), (u'part', 0.0007445082076301448), (u'deep_learning', 0.00074445635965077997), (u'know', 0.00074328766778288295), (u'regression', 0.00074301557810828452), (u'data_scientists', 0.00074238736623276921), (u'tools', 0.0007422152624720022), (u'make', 0.00074220790114136405), (u'software', 0.00074207241302705709), (u'getting', 0.00074184495858281509), (u'good', 0.00074176558547930655), (u'medium', 0.00074175450908493996), (u'artificial_intelligence', 0.00074017055929738473), (u'need', 0.00073868332883564953), (u'tensorflow', 0.00073827249478075716), (u'infrastructure', 0.00073818548072627881), (u'spark', 0.00073779684890874222), (u'big_data', 0.00073754679773540958)])\n"
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "'int' object has no attribute '__getitem__'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-17-0c1b44e6f3b8>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      6\u001b[0m     \u001b[0;31m#print top[1]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m     \u001b[0;31m#print \"\\n\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m     \u001b[0mwc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgenerate_from_frequencies\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mt\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtop\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      9\u001b[0m     \u001b[0mimg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mwc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_image\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m     \u001b[0mimg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'{0}.png'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: 'int' object has no attribute '__getitem__'"
     ]
    }
   ],
   "source": [
    "wc = WordCloud()\n",
    "topics = lda.show_topics(num_words=50,formatted=False)\n",
    "print topics[0]\n",
    "for i, top in enumerate(topics):\n",
    "    #print 'Topic:', i\n",
    "    #print top[1]\n",
    "    #print \"\\n\"\n",
    "    wc.generate_from_frequencies([(t[1],t[0]) for t in top])\n",
    "    img = wc.to_image()\n",
    "    img.save('{0}.png'.format(i))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}