Insights_potus_top_topics.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# %%\n",
    "#  Top Topics \n",
    "\n",
    "* model: (50t,600,10)\n",
    "* scale: 125"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd \n",
    "import numpy as np\n",
    "from helpers import potus\n",
    "import glob\n",
    "import re\n",
    "import os\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<module 'helpers.potus' from '/mnt/batch/tasks/shared/LS_root/mounts/clusters/ilab2-vm2/code/Users/mutazag/ilab2/helpers/potus.py'>"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import imp\n",
    "imp.reload(potus)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## (50t,600,10)\n",
    "\n",
    "Load results for scales 125 and 25."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "load lda model from /home/azureuser/cloudfiles/code/data/processing/potus/experiment/potus_50t_600_10/lda.model\n",
      "load bow from  /home/azureuser/cloudfiles/code/data/processing/potus/experiment/potus_50t_600_10/bow.pkl\n",
      "File exists and is readable\n",
      "/home/azureuser/cloudfiles/code/data/processing/potus/experiment/potus_50t_600_10/speeches_dfr_potus_50t_600_10_klds_100_100.pkl\n",
      "/home/azureuser/cloudfiles/code/data/processing/potus/experiment/potus_50t_600_10/speeches_dfr_potus_50t_600_10_klds_125_125.pkl\n",
      "/home/azureuser/cloudfiles/code/data/processing/potus/experiment/potus_50t_600_10/speeches_dfr_potus_50t_600_10_klds_25_25.pkl\n",
      "t: 50, Nw: 25, Tw:25\n",
      "962  before trim\n",
      "/home/azureuser/cloudfiles/code/data/processing/potus/experiment/potus_50t_600_10/speeches_dfr_potus_50t_600_10_klds_50_50.pkl\n",
      "/home/azureuser/cloudfiles/code/data/processing/potus/experiment/potus_50t_600_10/speeches_dfr_potus_50t_600_10_klds_75_75.pkl\n",
      "962\n",
      "1\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>kld_filename</th>\n",
       "      <th>topics</th>\n",
       "      <th>Nw</th>\n",
       "      <th>Tw</th>\n",
       "      <th>path</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>speeches_dfr_potus_50t_600_10_klds_25_25.pkl</td>\n",
       "      <td>50</td>\n",
       "      <td>25</td>\n",
       "      <td>25</td>\n",
       "      <td>/home/azureuser/cloudfiles/code/data/processin...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   kld_filename  topics  Nw  Tw  \\\n",
       "0  speeches_dfr_potus_50t_600_10_klds_25_25.pkl      50  25  25   \n",
       "\n",
       "                                                path  \n",
       "0  /home/azureuser/cloudfiles/code/data/processin...  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "Nw\n",
       "25    962\n",
       "Name: speech, dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "exp = 'potus_50t_600_10'\n",
    "scales = [25]\n",
    "result = potus.gridExpResult(exp, scales)\n",
    "\n",
    "print(len(result.df))\n",
    "print(len(result.settings))\n",
    "display(result.settings)\n",
    "display(result.df.groupby('Nw').speech.count())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Print Top Topics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['53.5% - Topic 04: object principle circumstance fellow_citizen',\n",
       " '18.9% - Topic 29: condition service case general',\n",
       " '7.2% - Topic 24: heart democracy liberty struggle',\n",
       " '6.6% - Topic 21: responsibility principle liberty justice',\n",
       " '5.5% - Topic 33: bill constitution provision case']"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "topics.get_top_topics_from_df(result.model, result.df.iloc[0].probs, ntopics=5, nterms=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from helpers.topics import get_topic\n",
    "\n",
    "ntopics = 5\n",
    "result_model = result.model\n",
    "\n",
    "\n",
    "\n",
    "iloc = 123\n",
    "\n",
    "df_weigths_matrix = pd.DataFrame()\n",
    "\n",
    "for idx, r in result.df.iterrows(): \n",
    "    speech_topic_probs = r.probs\n",
    "    speech_id = r.speech_id\n",
    "\n",
    "    topics_ordered = np.flip(speech_topic_probs.argsort())\n",
    "\n",
    "    dft =pd.Series(topics_ordered, name='topic').reset_index().rename(columns={'index': 'weight'})\n",
    "    dft.weight = np.maximum(0,ntopics-dft.weight)\n",
    "    dft['speech_id'] = speech_id\n",
    "    \n",
    "    df_weigths_matrix = df_weigths_matrix.append(dft)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>weight</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>topic</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>1819</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1644</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1136</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>956</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>942</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>766</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>687</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>638</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>595</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>567</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>432</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>422</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>393</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>354</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>281</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>244</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>242</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>232</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>224</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>224</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>192</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>174</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>174</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>172</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>163</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>120</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>91</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       weight\n",
       "topic        \n",
       "29       1819\n",
       "14       1644\n",
       "2        1136\n",
       "4         956\n",
       "44        942\n",
       "21        766\n",
       "12        687\n",
       "24        638\n",
       "27        595\n",
       "40        567\n",
       "23        432\n",
       "0         422\n",
       "33        393\n",
       "31        354\n",
       "28        281\n",
       "10        244\n",
       "3         242\n",
       "42        232\n",
       "8         224\n",
       "19        224\n",
       "5         192\n",
       "34        174\n",
       "38        174\n",
       "49        172\n",
       "32        163\n",
       "16        120\n",
       "26         91\n",
       "20         60\n",
       "43         59\n",
       "45         52\n",
       "22         47\n",
       "39         46\n",
       "46         43\n",
       "36         42\n",
       "17         41\n",
       "48         35\n",
       "6          31\n",
       "18         21\n",
       "35         18\n",
       "15         13\n",
       "47         12\n",
       "11         10\n",
       "13          8\n",
       "1           5\n",
       "9           3\n",
       "37          0\n",
       "7           0\n",
       "41          0\n",
       "30          0\n",
       "25          0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "14430"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_topics_by_weight = df_weigths_matrix.groupby('topic').sum('weight').sort_values('weight', ascending=False)\n",
    "display(top_topics_by_weight)\n",
    "\n",
    "sum(top_topics_by_weight.weight)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7.1 64-bit ('base': conda)",
   "language": "python",
   "name": "python37164bitbasecondabc65611516f5429da066830f275de3ab"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}