slides_topicmodeling_simple.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Playground for Topic Modeling slides - Simple Version\n",
    "- Stephen W. Thomas\n",
    "- Used for MMA 865; MMAI 891; Exec Ed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime\n",
    "print(datetime.datetime.now())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "InteractiveShell.ast_node_interactivity = \"all\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Read in the Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 6818 entries, 0 to 6817\n",
      "Data columns (total 9 columns):\n",
      "loan_id        6818 non-null int64\n",
      "status         6818 non-null object\n",
      "sector         6818 non-null object\n",
      "en             6818 non-null object\n",
      "country        6818 non-null object\n",
      "gender         6818 non-null object\n",
      "loan_amount    6818 non-null float64\n",
      "nonpayment     6818 non-null object\n",
      "en_clean       6802 non-null object\n",
      "dtypes: float64(1), int64(1), object(7)\n",
      "memory usage: 479.5+ KB\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 6802 entries, 0 to 6801\n",
      "Data columns (total 9 columns):\n",
      "loan_id        6802 non-null int64\n",
      "status         6802 non-null object\n",
      "sector         6802 non-null object\n",
      "en             6802 non-null object\n",
      "country        6802 non-null object\n",
      "gender         6802 non-null object\n",
      "loan_amount    6802 non-null float64\n",
      "nonpayment     6802 non-null object\n",
      "en_clean       6802 non-null object\n",
      "dtypes: float64(1), int64(1), object(7)\n",
      "memory usage: 478.3+ KB\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(6802, 9)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>loan_id</th>\n",
       "      <th>status</th>\n",
       "      <th>sector</th>\n",
       "      <th>en</th>\n",
       "      <th>country</th>\n",
       "      <th>gender</th>\n",
       "      <th>loan_amount</th>\n",
       "      <th>nonpayment</th>\n",
       "      <th>en_clean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>defaulted</td>\n",
       "      <td>Agriculture</td>\n",
       "      <td>&lt;i&gt;This description is written by Rowland Amul...</td>\n",
       "      <td>Kenya</td>\n",
       "      <td>M</td>\n",
       "      <td>500.0</td>\n",
       "      <td>lender</td>\n",
       "      <td>Robert, 40, is married and has 6 children. In ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>defaulted</td>\n",
       "      <td>Food</td>\n",
       "      <td>&lt;i&gt;This description is written by Rowland Amul...</td>\n",
       "      <td>Kenya</td>\n",
       "      <td>F</td>\n",
       "      <td>500.0</td>\n",
       "      <td>lender</td>\n",
       "      <td>Petronilla, 30, was deserted by her husband an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>defaulted</td>\n",
       "      <td>Food</td>\n",
       "      <td>&lt;i&gt;This description was written by Richard Maz...</td>\n",
       "      <td>Kenya</td>\n",
       "      <td>M</td>\n",
       "      <td>500.0</td>\n",
       "      <td>lender</td>\n",
       "      <td>Tom Mung'ahu, 45, is married and has 6 childre...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>defaulted</td>\n",
       "      <td>Services</td>\n",
       "      <td>&lt;i&gt;This description was written by Rowland Amu...</td>\n",
       "      <td>Kenya</td>\n",
       "      <td>F</td>\n",
       "      <td>500.0</td>\n",
       "      <td>lender</td>\n",
       "      <td>Benedina, 42, is married and has 4 girls. In a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>defaulted</td>\n",
       "      <td>Construction</td>\n",
       "      <td>&lt;i&gt;This description was written by Rowland Amu...</td>\n",
       "      <td>Kenya</td>\n",
       "      <td>M</td>\n",
       "      <td>500.0</td>\n",
       "      <td>lender</td>\n",
       "      <td>Vincent Ondego 40, is married and has 8 childr...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   loan_id     status        sector  \\\n",
       "0        0  defaulted   Agriculture   \n",
       "1        1  defaulted          Food   \n",
       "2        2  defaulted          Food   \n",
       "3        3  defaulted      Services   \n",
       "4        4  defaulted  Construction   \n",
       "\n",
       "                                                  en country gender  \\\n",
       "0  <i>This description is written by Rowland Amul...   Kenya      M   \n",
       "1  <i>This description is written by Rowland Amul...   Kenya      F   \n",
       "2  <i>This description was written by Richard Maz...   Kenya      M   \n",
       "3  <i>This description was written by Rowland Amu...   Kenya      F   \n",
       "4  <i>This description was written by Rowland Amu...   Kenya      M   \n",
       "\n",
       "   loan_amount nonpayment                                           en_clean  \n",
       "0        500.0     lender  Robert, 40, is married and has 6 children. In ...  \n",
       "1        500.0     lender  Petronilla, 30, was deserted by her husband an...  \n",
       "2        500.0     lender  Tom Mung'ahu, 45, is married and has 6 childre...  \n",
       "3        500.0     lender  Benedina, 42, is married and has 4 girls. In a...  \n",
       "4        500.0     lender  Vincent Ondego 40, is married and has 8 childr...  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>loan_id</th>\n",
       "      <th>status</th>\n",
       "      <th>sector</th>\n",
       "      <th>en</th>\n",
       "      <th>country</th>\n",
       "      <th>gender</th>\n",
       "      <th>loan_amount</th>\n",
       "      <th>nonpayment</th>\n",
       "      <th>en_clean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6797</th>\n",
       "      <td>7983</td>\n",
       "      <td>paid</td>\n",
       "      <td>Clothing</td>\n",
       "      <td>Rosa is a member of the 3 de Mayo Communal Ban...</td>\n",
       "      <td>Ecuador</td>\n",
       "      <td>F</td>\n",
       "      <td>1050.0</td>\n",
       "      <td>lender</td>\n",
       "      <td>Rosa is a member of the 3 de Mayo Communal Ban...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6798</th>\n",
       "      <td>7984</td>\n",
       "      <td>paid</td>\n",
       "      <td>Agriculture</td>\n",
       "      <td>The communal bank, \"Cumbe,\" is located on the ...</td>\n",
       "      <td>Ecuador</td>\n",
       "      <td>F</td>\n",
       "      <td>2650.0</td>\n",
       "      <td>partner</td>\n",
       "      <td>The communal bank, \"Cumbe,\" is located on the ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6799</th>\n",
       "      <td>7985</td>\n",
       "      <td>paid</td>\n",
       "      <td>Food</td>\n",
       "      <td>Gladys Burgos Macas is a member of the 3 de Ma...</td>\n",
       "      <td>Ecuador</td>\n",
       "      <td>F</td>\n",
       "      <td>775.0</td>\n",
       "      <td>partner</td>\n",
       "      <td>Gladys Burgos Macas is a member of the 3 de Ma...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6800</th>\n",
       "      <td>7986</td>\n",
       "      <td>paid</td>\n",
       "      <td>Retail</td>\n",
       "      <td>The Communal Bank &lt;em&gt;Las Caleras&lt;/em&gt; is loca...</td>\n",
       "      <td>Ecuador</td>\n",
       "      <td>F</td>\n",
       "      <td>275.0</td>\n",
       "      <td>partner</td>\n",
       "      <td>The Communal Bank Las Caleras is located in th...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6801</th>\n",
       "      <td>7987</td>\n",
       "      <td>paid</td>\n",
       "      <td>Services</td>\n",
       "      <td>Eliud Ouma owns the Purple Hut salon located o...</td>\n",
       "      <td>Kenya</td>\n",
       "      <td>M</td>\n",
       "      <td>275.0</td>\n",
       "      <td>partner</td>\n",
       "      <td>Eliud Ouma owns the Purple Hut salon located o...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      loan_id status       sector  \\\n",
       "6797     7983   paid     Clothing   \n",
       "6798     7984   paid  Agriculture   \n",
       "6799     7985   paid         Food   \n",
       "6800     7986   paid       Retail   \n",
       "6801     7987   paid     Services   \n",
       "\n",
       "                                                     en  country gender  \\\n",
       "6797  Rosa is a member of the 3 de Mayo Communal Ban...  Ecuador      F   \n",
       "6798  The communal bank, \"Cumbe,\" is located on the ...  Ecuador      F   \n",
       "6799  Gladys Burgos Macas is a member of the 3 de Ma...  Ecuador      F   \n",
       "6800  The Communal Bank <em>Las Caleras</em> is loca...  Ecuador      F   \n",
       "6801  Eliud Ouma owns the Purple Hut salon located o...    Kenya      M   \n",
       "\n",
       "      loan_amount nonpayment  \\\n",
       "6797       1050.0     lender   \n",
       "6798       2650.0    partner   \n",
       "6799        775.0    partner   \n",
       "6800        275.0    partner   \n",
       "6801        275.0    partner   \n",
       "\n",
       "                                               en_clean  \n",
       "6797  Rosa is a member of the 3 de Mayo Communal Ban...  \n",
       "6798  The communal bank, \"Cumbe,\" is located on the ...  \n",
       "6799  Gladys Burgos Macas is a member of the 3 de Ma...  \n",
       "6800  The Communal Bank Las Caleras is located in th...  \n",
       "6801  Eliud Ouma owns the Purple Hut salon located o...  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kiva_df=pd.read_csv(\"data/kiva_cleaned.csv\")\n",
    "kiva_df.info()\n",
    "\n",
    "kiva_df = kiva_df.dropna()\n",
    "kiva_df = kiva_df.reset_index(drop=True)\n",
    "\n",
    "kiva_df.info()\n",
    "kiva_df.shape\n",
    "kiva_df.head()\n",
    "kiva_df.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Text Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.corpus import stopwords \n",
    "from nltk.stem import WordNetLemmatizer\n",
    "import re\n",
    "import unidecode\n",
    "\n",
    "#import warnings\n",
    "\n",
    "#with warnings.catch_warnings():\n",
    "#    warnings.filterwarnings(\"ignore\",category=DeprecationWarning)\n",
    "\n",
    "stop_words = set(stopwords.words('english') + stopwords.words('spanish'))\n",
    "\n",
    "lemmer = WordNetLemmatizer()\n",
    "\n",
    "def preprocess(x):\n",
    "    # Lower case\n",
    "    x = x.lower()\n",
    "    \n",
    "    # Remove punctuation\n",
    "    x = re.sub(r'[^\\w\\s]', '', x)\n",
    "    \n",
    "    # Remove non-unicode\n",
    "    x = unidecode.unidecode(x)\n",
    "    \n",
    "    # Remove numbers\n",
    "    x = re.sub(r'\\d+', '', x)\n",
    "    \n",
    "    # Remove stopwords and lemmatize\n",
    "    x = [lemmer.lemmatize(w) for w in x.split() if w not in stop_words]\n",
    "    return ' '.join(x) \n",
    "\n",
    "kiva_df['en_clean_pre'] = kiva_df['en_clean'].apply(preprocess)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>loan_id</th>\n",
       "      <th>status</th>\n",
       "      <th>sector</th>\n",
       "      <th>en</th>\n",
       "      <th>country</th>\n",
       "      <th>gender</th>\n",
       "      <th>loan_amount</th>\n",
       "      <th>nonpayment</th>\n",
       "      <th>en_clean</th>\n",
       "      <th>en_clean_pre</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>defaulted</td>\n",
       "      <td>Agriculture</td>\n",
       "      <td>&lt;i&gt;This description is written by Rowland Amul...</td>\n",
       "      <td>Kenya</td>\n",
       "      <td>M</td>\n",
       "      <td>500.0</td>\n",
       "      <td>lender</td>\n",
       "      <td>Robert, 40, is married and has 6 children. In ...</td>\n",
       "      <td>robert married child addition family take care...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>defaulted</td>\n",
       "      <td>Food</td>\n",
       "      <td>&lt;i&gt;This description is written by Rowland Amul...</td>\n",
       "      <td>Kenya</td>\n",
       "      <td>F</td>\n",
       "      <td>500.0</td>\n",
       "      <td>lender</td>\n",
       "      <td>Petronilla, 30, was deserted by her husband an...</td>\n",
       "      <td>petronilla deserted husband responsible upbrin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>defaulted</td>\n",
       "      <td>Food</td>\n",
       "      <td>&lt;i&gt;This description was written by Richard Maz...</td>\n",
       "      <td>Kenya</td>\n",
       "      <td>M</td>\n",
       "      <td>500.0</td>\n",
       "      <td>lender</td>\n",
       "      <td>Tom Mung'ahu, 45, is married and has 6 childre...</td>\n",
       "      <td>tom mungahu married child child attending elem...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>defaulted</td>\n",
       "      <td>Services</td>\n",
       "      <td>&lt;i&gt;This description was written by Rowland Amu...</td>\n",
       "      <td>Kenya</td>\n",
       "      <td>F</td>\n",
       "      <td>500.0</td>\n",
       "      <td>lender</td>\n",
       "      <td>Benedina, 42, is married and has 4 girls. In a...</td>\n",
       "      <td>benedina married girl addition family also tak...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>defaulted</td>\n",
       "      <td>Construction</td>\n",
       "      <td>&lt;i&gt;This description was written by Rowland Amu...</td>\n",
       "      <td>Kenya</td>\n",
       "      <td>M</td>\n",
       "      <td>500.0</td>\n",
       "      <td>lender</td>\n",
       "      <td>Vincent Ondego 40, is married and has 8 childr...</td>\n",
       "      <td>vincent ondego married child beside family vin...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   loan_id     status        sector  \\\n",
       "0        0  defaulted   Agriculture   \n",
       "1        1  defaulted          Food   \n",
       "2        2  defaulted          Food   \n",
       "3        3  defaulted      Services   \n",
       "4        4  defaulted  Construction   \n",
       "\n",
       "                                                  en country gender  \\\n",
       "0  <i>This description is written by Rowland Amul...   Kenya      M   \n",
       "1  <i>This description is written by Rowland Amul...   Kenya      F   \n",
       "2  <i>This description was written by Richard Maz...   Kenya      M   \n",
       "3  <i>This description was written by Rowland Amu...   Kenya      F   \n",
       "4  <i>This description was written by Rowland Amu...   Kenya      M   \n",
       "\n",
       "   loan_amount nonpayment                                           en_clean  \\\n",
       "0        500.0     lender  Robert, 40, is married and has 6 children. In ...   \n",
       "1        500.0     lender  Petronilla, 30, was deserted by her husband an...   \n",
       "2        500.0     lender  Tom Mung'ahu, 45, is married and has 6 childre...   \n",
       "3        500.0     lender  Benedina, 42, is married and has 4 girls. In a...   \n",
       "4        500.0     lender  Vincent Ondego 40, is married and has 8 childr...   \n",
       "\n",
       "                                        en_clean_pre  \n",
       "0  robert married child addition family take care...  \n",
       "1  petronilla deserted husband responsible upbrin...  \n",
       "2  tom mungahu married child child attending elem...  \n",
       "3  benedina married girl addition family also tak...  \n",
       "4  vincent ondego married child beside family vin...  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kiva_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Robert, 40, is married and has 6 children. In addition to his family of 8, he takes care of his mother and 5 brothers. Robert started by planting vegetables and selling at the local market. He then diversified with a tea nursery which is more profitable. Given a loan of $500, Robert will be able to improve his activities by buying fertilizers, pesticides, a pump and seedlings. The pump will remove the uncertainties of the weather. The current capital in his venture is Ksh 10500 or $150, which does not allow Robert to maximize his potential. Robert completed high school and has never got employment. He got apprentice training from a tea extension officer. He is a go getter, and his main hobby is teaching music.'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kiva_df.iloc[0,:].en_clean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'robert married child addition family take care mother brother robert started planting vegetable selling local market diversified tea nursery profitable given loan robert able improve activity buying fertilizer pesticide pump seedling pump remove uncertainty weather current capital venture ksh allow robert maximize potential robert completed high school never got employment got apprentice training tea extension officer go getter main hobby teaching music'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kiva_df.iloc[0,:].en_clean_pre"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Topic Modeling with Sci-kit Learn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(6802, 270)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "vectorizer = TfidfVectorizer(max_df=0.5, min_df=0.05, \n",
    "                             max_features=1000, ngram_range=[1,3])\n",
    "dtm = vectorizer.fit_transform(kiva_df['en_clean_pre'])\n",
    "print(dtm.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['able',\n",
       " 'active',\n",
       " 'active member',\n",
       " 'activity',\n",
       " 'add',\n",
       " 'additional',\n",
       " 'age',\n",
       " 'ago',\n",
       " 'allow',\n",
       " 'also',\n",
       " 'always',\n",
       " 'amount',\n",
       " 'another',\n",
       " 'applied',\n",
       " 'applied loan',\n",
       " 'area',\n",
       " 'attend',\n",
       " 'back',\n",
       " 'basic',\n",
       " 'bean',\n",
       " 'began',\n",
       " 'belief',\n",
       " 'better',\n",
       " 'born',\n",
       " 'bought',\n",
       " 'boy',\n",
       " 'business selling',\n",
       " 'buy',\n",
       " 'buying',\n",
       " 'capital',\n",
       " 'care',\n",
       " 'cement',\n",
       " 'child school',\n",
       " 'childrens',\n",
       " 'church',\n",
       " 'city',\n",
       " 'client',\n",
       " 'clothes',\n",
       " 'clothing',\n",
       " 'college',\n",
       " 'community',\n",
       " 'continue',\n",
       " 'cost',\n",
       " 'could',\n",
       " 'cow',\n",
       " 'currently',\n",
       " 'customer',\n",
       " 'dairy',\n",
       " 'dairy cow',\n",
       " 'daughter',\n",
       " 'day',\n",
       " 'decided',\n",
       " 'demand',\n",
       " 'different',\n",
       " 'domingo',\n",
       " 'dominican',\n",
       " 'dream',\n",
       " 'due',\n",
       " 'earn',\n",
       " 'educate',\n",
       " 'education',\n",
       " 'enable',\n",
       " 'enough',\n",
       " 'entrepreneur',\n",
       " 'every',\n",
       " 'every day',\n",
       " 'expand',\n",
       " 'expand business',\n",
       " 'expense',\n",
       " 'experience',\n",
       " 'explains',\n",
       " 'family',\n",
       " 'farm',\n",
       " 'farmer',\n",
       " 'farming',\n",
       " 'father',\n",
       " 'fee',\n",
       " 'feed',\n",
       " 'first',\n",
       " 'first loan',\n",
       " 'five',\n",
       " 'five child',\n",
       " 'food',\n",
       " 'four',\n",
       " 'four child',\n",
       " 'friend',\n",
       " 'fruit',\n",
       " 'fund',\n",
       " 'future',\n",
       " 'get',\n",
       " 'girl',\n",
       " 'give',\n",
       " 'given',\n",
       " 'go',\n",
       " 'goal',\n",
       " 'good',\n",
       " 'great',\n",
       " 'grocery',\n",
       " 'group',\n",
       " 'grow',\n",
       " 'growing',\n",
       " 'hard',\n",
       " 'hardworking',\n",
       " 'help',\n",
       " 'high',\n",
       " 'home',\n",
       " 'hope',\n",
       " 'house',\n",
       " 'household',\n",
       " 'husband',\n",
       " 'improve',\n",
       " 'income',\n",
       " 'increase',\n",
       " 'intends',\n",
       " 'inventory',\n",
       " 'invest',\n",
       " 'item',\n",
       " 'job',\n",
       " 'keep',\n",
       " 'kenya',\n",
       " 'kiva',\n",
       " 'lack',\n",
       " 'land',\n",
       " 'large',\n",
       " 'last',\n",
       " 'left',\n",
       " 'life',\n",
       " 'like',\n",
       " 'little',\n",
       " 'live',\n",
       " 'living',\n",
       " 'loan buy',\n",
       " 'loan purchase',\n",
       " 'loan usd',\n",
       " 'local',\n",
       " 'located',\n",
       " 'lot',\n",
       " 'made',\n",
       " 'main',\n",
       " 'maize',\n",
       " 'make',\n",
       " 'making',\n",
       " 'many',\n",
       " 'market',\n",
       " 'married',\n",
       " 'married child',\n",
       " 'material',\n",
       " 'meet',\n",
       " 'member',\n",
       " 'merchandise',\n",
       " 'milk',\n",
       " 'money',\n",
       " 'month',\n",
       " 'monthly',\n",
       " 'mother',\n",
       " 'much',\n",
       " 'nairobi',\n",
       " 'need',\n",
       " 'need loan',\n",
       " 'neighborhood',\n",
       " 'new',\n",
       " 'next',\n",
       " 'offer',\n",
       " 'old',\n",
       " 'old married',\n",
       " 'one',\n",
       " 'open',\n",
       " 'operates',\n",
       " 'opportunity',\n",
       " 'order',\n",
       " 'owns',\n",
       " 'parent',\n",
       " 'part',\n",
       " 'pay',\n",
       " 'people',\n",
       " 'per',\n",
       " 'period',\n",
       " 'person',\n",
       " 'plan',\n",
       " 'pm',\n",
       " 'price',\n",
       " 'primary',\n",
       " 'primary school',\n",
       " 'produce',\n",
       " 'product',\n",
       " 'profit',\n",
       " 'provide',\n",
       " 'purchase',\n",
       " 'quality',\n",
       " 'receive',\n",
       " 'rent',\n",
       " 'repay',\n",
       " 'repay loan',\n",
       " 'requesting',\n",
       " 'requesting loan',\n",
       " 'rice',\n",
       " 'run',\n",
       " 'running',\n",
       " 'sale',\n",
       " 'santo',\n",
       " 'santo domingo',\n",
       " 'save',\n",
       " 'saving',\n",
       " 'say',\n",
       " 'school',\n",
       " 'school fee',\n",
       " 'season',\n",
       " 'second',\n",
       " 'second loan',\n",
       " 'secondary',\n",
       " 'secondary school',\n",
       " 'selling',\n",
       " 'service',\n",
       " 'shoe',\n",
       " 'shop',\n",
       " 'since',\n",
       " 'single',\n",
       " 'single mother',\n",
       " 'six',\n",
       " 'small',\n",
       " 'small business',\n",
       " 'sold',\n",
       " 'source',\n",
       " 'source income',\n",
       " 'start',\n",
       " 'started',\n",
       " 'started business',\n",
       " 'still',\n",
       " 'stock',\n",
       " 'store',\n",
       " 'study',\n",
       " 'supply',\n",
       " 'support',\n",
       " 'support family',\n",
       " 'take',\n",
       " 'take care',\n",
       " 'therefore',\n",
       " 'three',\n",
       " 'three child',\n",
       " 'time',\n",
       " 'together',\n",
       " 'town',\n",
       " 'training',\n",
       " 'two',\n",
       " 'two child',\n",
       " 'two year',\n",
       " 'usd',\n",
       " 'use',\n",
       " 'use loan',\n",
       " 'used',\n",
       " 'variety',\n",
       " 'vegetable',\n",
       " 'want',\n",
       " 'way',\n",
       " 'week',\n",
       " 'well',\n",
       " 'wholesale',\n",
       " 'wife',\n",
       " 'within',\n",
       " 'woman',\n",
       " 'woman group',\n",
       " 'work',\n",
       " 'working',\n",
       " 'would',\n",
       " 'would like',\n",
       " 'year ago',\n",
       " 'year old',\n",
       " 'year old married',\n",
       " 'yearold',\n",
       " 'young']"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer.get_feature_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.decomposition import LatentDirichletAllocation\n",
    "\n",
    "lda_model = LatentDirichletAllocation(n_components=20,\n",
    "                                      doc_topic_prior=None,\n",
    "                                      topic_word_prior=None,\n",
    "                                      max_iter=200, \n",
    "                                      learning_method='batch', \n",
    "                                      random_state=123,\n",
    "                                      n_jobs=2,\n",
    "                                      verbose=0)\n",
    "lda_output = lda_model.fit(dtm)\n",
    "\n",
    "# Log Likelyhood: Higher the better\n",
    "ll = lda_model.score(dtm)\n",
    "\n",
    "# Perplexity: Lower the better. Perplexity = exp(-1. * log-likelihood per word)\n",
    "perp = lda_model.perplexity(dtm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Theta = document-topic matrix\n",
    "# Beta = components_ = topic-term matrix\n",
    "theta = pd.DataFrame(lda_model.transform(dtm))\n",
    "beta = pd.DataFrame(lda_model.components_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "      <th>18</th>\n",
       "      <th>19</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.008651</td>\n",
       "      <td>0.008651</td>\n",
       "      <td>0.008651</td>\n",
       "      <td>0.008651</td>\n",
       "      <td>0.483942</td>\n",
       "      <td>0.008651</td>\n",
       "      <td>0.008651</td>\n",
       "      <td>0.008651</td>\n",
       "      <td>0.008651</td>\n",
       "      <td>0.008651</td>\n",
       "      <td>0.008651</td>\n",
       "      <td>0.008651</td>\n",
       "      <td>0.360335</td>\n",
       "      <td>0.008651</td>\n",
       "      <td>0.008651</td>\n",
       "      <td>0.008651</td>\n",
       "      <td>0.008651</td>\n",
       "      <td>0.008651</td>\n",
       "      <td>0.008651</td>\n",
       "      <td>0.008651</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.007398</td>\n",
       "      <td>0.007398</td>\n",
       "      <td>0.007398</td>\n",
       "      <td>0.007398</td>\n",
       "      <td>0.007398</td>\n",
       "      <td>0.859442</td>\n",
       "      <td>0.007398</td>\n",
       "      <td>0.007398</td>\n",
       "      <td>0.007398</td>\n",
       "      <td>0.007398</td>\n",
       "      <td>0.007398</td>\n",
       "      <td>0.007398</td>\n",
       "      <td>0.007398</td>\n",
       "      <td>0.007398</td>\n",
       "      <td>0.007398</td>\n",
       "      <td>0.007398</td>\n",
       "      <td>0.007398</td>\n",
       "      <td>0.007398</td>\n",
       "      <td>0.007398</td>\n",
       "      <td>0.007398</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.008250</td>\n",
       "      <td>0.008250</td>\n",
       "      <td>0.008250</td>\n",
       "      <td>0.008250</td>\n",
       "      <td>0.008250</td>\n",
       "      <td>0.008250</td>\n",
       "      <td>0.008250</td>\n",
       "      <td>0.008250</td>\n",
       "      <td>0.008250</td>\n",
       "      <td>0.008250</td>\n",
       "      <td>0.008250</td>\n",
       "      <td>0.008250</td>\n",
       "      <td>0.008250</td>\n",
       "      <td>0.008250</td>\n",
       "      <td>0.008250</td>\n",
       "      <td>0.008250</td>\n",
       "      <td>0.658675</td>\n",
       "      <td>0.192827</td>\n",
       "      <td>0.008250</td>\n",
       "      <td>0.008250</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.010299</td>\n",
       "      <td>0.010299</td>\n",
       "      <td>0.010299</td>\n",
       "      <td>0.010299</td>\n",
       "      <td>0.010299</td>\n",
       "      <td>0.010299</td>\n",
       "      <td>0.010299</td>\n",
       "      <td>0.010299</td>\n",
       "      <td>0.010299</td>\n",
       "      <td>0.010299</td>\n",
       "      <td>0.010299</td>\n",
       "      <td>0.010299</td>\n",
       "      <td>0.804310</td>\n",
       "      <td>0.010299</td>\n",
       "      <td>0.010299</td>\n",
       "      <td>0.010299</td>\n",
       "      <td>0.010299</td>\n",
       "      <td>0.010299</td>\n",
       "      <td>0.010299</td>\n",
       "      <td>0.010299</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.821479</td>\n",
       "      <td>0.009396</td>\n",
       "      <td>0.009396</td>\n",
       "      <td>0.009396</td>\n",
       "      <td>0.009396</td>\n",
       "      <td>0.009396</td>\n",
       "      <td>0.009396</td>\n",
       "      <td>0.009396</td>\n",
       "      <td>0.009396</td>\n",
       "      <td>0.009396</td>\n",
       "      <td>0.009396</td>\n",
       "      <td>0.009396</td>\n",
       "      <td>0.009396</td>\n",
       "      <td>0.009396</td>\n",
       "      <td>0.009396</td>\n",
       "      <td>0.009396</td>\n",
       "      <td>0.009396</td>\n",
       "      <td>0.009396</td>\n",
       "      <td>0.009396</td>\n",
       "      <td>0.009396</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         0         1         2         3         4         5         6   \\\n",
       "0  0.008651  0.008651  0.008651  0.008651  0.483942  0.008651  0.008651   \n",
       "1  0.007398  0.007398  0.007398  0.007398  0.007398  0.859442  0.007398   \n",
       "2  0.008250  0.008250  0.008250  0.008250  0.008250  0.008250  0.008250   \n",
       "3  0.010299  0.010299  0.010299  0.010299  0.010299  0.010299  0.010299   \n",
       "4  0.821479  0.009396  0.009396  0.009396  0.009396  0.009396  0.009396   \n",
       "\n",
       "         7         8         9         10        11        12        13  \\\n",
       "0  0.008651  0.008651  0.008651  0.008651  0.008651  0.360335  0.008651   \n",
       "1  0.007398  0.007398  0.007398  0.007398  0.007398  0.007398  0.007398   \n",
       "2  0.008250  0.008250  0.008250  0.008250  0.008250  0.008250  0.008250   \n",
       "3  0.010299  0.010299  0.010299  0.010299  0.010299  0.804310  0.010299   \n",
       "4  0.009396  0.009396  0.009396  0.009396  0.009396  0.009396  0.009396   \n",
       "\n",
       "         14        15        16        17        18        19  \n",
       "0  0.008651  0.008651  0.008651  0.008651  0.008651  0.008651  \n",
       "1  0.007398  0.007398  0.007398  0.007398  0.007398  0.007398  \n",
       "2  0.008250  0.008250  0.658675  0.192827  0.008250  0.008250  \n",
       "3  0.010299  0.010299  0.010299  0.010299  0.010299  0.010299  \n",
       "4  0.009396  0.009396  0.009396  0.009396  0.009396  0.009396  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "theta.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>260</th>\n",
       "      <th>261</th>\n",
       "      <th>262</th>\n",
       "      <th>263</th>\n",
       "      <th>264</th>\n",
       "      <th>265</th>\n",
       "      <th>266</th>\n",
       "      <th>267</th>\n",
       "      <th>268</th>\n",
       "      <th>269</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>28.450467</td>\n",
       "      <td>55.671889</td>\n",
       "      <td>49.078044</td>\n",
       "      <td>20.139093</td>\n",
       "      <td>10.131653</td>\n",
       "      <td>2.238602</td>\n",
       "      <td>4.844017</td>\n",
       "      <td>20.392001</td>\n",
       "      <td>0.379399</td>\n",
       "      <td>32.240180</td>\n",
       "      <td>...</td>\n",
       "      <td>75.758588</td>\n",
       "      <td>19.303593</td>\n",
       "      <td>8.688853</td>\n",
       "      <td>12.839707</td>\n",
       "      <td>11.922191</td>\n",
       "      <td>21.248398</td>\n",
       "      <td>27.594151</td>\n",
       "      <td>33.102967</td>\n",
       "      <td>0.050000</td>\n",
       "      <td>13.793507</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>12.547361</td>\n",
       "      <td>0.050000</td>\n",
       "      <td>0.050000</td>\n",
       "      <td>0.050000</td>\n",
       "      <td>0.050000</td>\n",
       "      <td>0.050000</td>\n",
       "      <td>0.314016</td>\n",
       "      <td>0.050000</td>\n",
       "      <td>0.050000</td>\n",
       "      <td>0.050000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.050000</td>\n",
       "      <td>0.050000</td>\n",
       "      <td>0.050000</td>\n",
       "      <td>0.050000</td>\n",
       "      <td>0.050000</td>\n",
       "      <td>0.050000</td>\n",
       "      <td>0.050000</td>\n",
       "      <td>0.050000</td>\n",
       "      <td>2.543769</td>\n",
       "      <td>0.050000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>12.299628</td>\n",
       "      <td>24.208785</td>\n",
       "      <td>22.185490</td>\n",
       "      <td>7.702309</td>\n",
       "      <td>4.612913</td>\n",
       "      <td>2.409955</td>\n",
       "      <td>3.894666</td>\n",
       "      <td>6.478088</td>\n",
       "      <td>0.479245</td>\n",
       "      <td>19.723512</td>\n",
       "      <td>...</td>\n",
       "      <td>24.990815</td>\n",
       "      <td>4.740272</td>\n",
       "      <td>4.946245</td>\n",
       "      <td>6.660111</td>\n",
       "      <td>5.612970</td>\n",
       "      <td>6.710831</td>\n",
       "      <td>13.696085</td>\n",
       "      <td>16.584664</td>\n",
       "      <td>0.050000</td>\n",
       "      <td>4.169932</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>38.018599</td>\n",
       "      <td>0.625488</td>\n",
       "      <td>0.050000</td>\n",
       "      <td>12.383906</td>\n",
       "      <td>2.379312</td>\n",
       "      <td>8.130464</td>\n",
       "      <td>42.873529</td>\n",
       "      <td>5.346212</td>\n",
       "      <td>12.875778</td>\n",
       "      <td>22.203624</td>\n",
       "      <td>...</td>\n",
       "      <td>5.003734</td>\n",
       "      <td>19.526729</td>\n",
       "      <td>13.453686</td>\n",
       "      <td>26.031663</td>\n",
       "      <td>14.001482</td>\n",
       "      <td>3.809130</td>\n",
       "      <td>13.377419</td>\n",
       "      <td>0.050000</td>\n",
       "      <td>13.818494</td>\n",
       "      <td>20.742405</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>31.489722</td>\n",
       "      <td>8.511836</td>\n",
       "      <td>8.250891</td>\n",
       "      <td>1.976989</td>\n",
       "      <td>1.115150</td>\n",
       "      <td>1.790282</td>\n",
       "      <td>2.074560</td>\n",
       "      <td>2.987586</td>\n",
       "      <td>0.050000</td>\n",
       "      <td>23.104390</td>\n",
       "      <td>...</td>\n",
       "      <td>1.604420</td>\n",
       "      <td>6.378504</td>\n",
       "      <td>2.433041</td>\n",
       "      <td>1.958979</td>\n",
       "      <td>0.050000</td>\n",
       "      <td>3.182817</td>\n",
       "      <td>2.796047</td>\n",
       "      <td>1.714359</td>\n",
       "      <td>1.312406</td>\n",
       "      <td>6.922944</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 270 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         0          1          2          3          4         5          6    \\\n",
       "0  28.450467  55.671889  49.078044  20.139093  10.131653  2.238602   4.844017   \n",
       "1  12.547361   0.050000   0.050000   0.050000   0.050000  0.050000   0.314016   \n",
       "2  12.299628  24.208785  22.185490   7.702309   4.612913  2.409955   3.894666   \n",
       "3  38.018599   0.625488   0.050000  12.383906   2.379312  8.130464  42.873529   \n",
       "4  31.489722   8.511836   8.250891   1.976989   1.115150  1.790282   2.074560   \n",
       "\n",
       "         7          8          9    ...        260        261        262  \\\n",
       "0  20.392001   0.379399  32.240180  ...  75.758588  19.303593   8.688853   \n",
       "1   0.050000   0.050000   0.050000  ...   0.050000   0.050000   0.050000   \n",
       "2   6.478088   0.479245  19.723512  ...  24.990815   4.740272   4.946245   \n",
       "3   5.346212  12.875778  22.203624  ...   5.003734  19.526729  13.453686   \n",
       "4   2.987586   0.050000  23.104390  ...   1.604420   6.378504   2.433041   \n",
       "\n",
       "         263        264        265        266        267        268        269  \n",
       "0  12.839707  11.922191  21.248398  27.594151  33.102967   0.050000  13.793507  \n",
       "1   0.050000   0.050000   0.050000   0.050000   0.050000   2.543769   0.050000  \n",
       "2   6.660111   5.612970   6.710831  13.696085  16.584664   0.050000   4.169932  \n",
       "3  26.031663  14.001482   3.809130  13.377419   0.050000  13.818494  20.742405  \n",
       "4   1.958979   0.050000   3.182817   2.796047   1.714359   1.312406   6.922944  \n",
       "\n",
       "[5 rows x 270 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "beta.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build Topic Summary\n",
    "feature_names = vectorizer.get_feature_names()\n",
    "weight = theta.sum(axis=0)\n",
    "support50 = (theta > 0.5).sum(axis=0)\n",
    "support10 = (theta > 0.1).sum(axis=0)\n",
    "termss = list()\n",
    "for topic_id, topic in enumerate(lda_model.components_):\n",
    "    terms = \" \".join([feature_names[i] for i in topic.argsort()[:-6 - 1:-1]])\n",
    "    termss.append(terms)\n",
    "topic_summary = pd.DataFrame({'TopicID': range(0, len(termss)), \"Support50\": support50, \"Support10\": support10, \"Weight\": weight, \"Terms\": termss})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TopicID</th>\n",
       "      <th>Support50</th>\n",
       "      <th>Support10</th>\n",
       "      <th>Weight</th>\n",
       "      <th>Terms</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>700</td>\n",
       "      <td>789</td>\n",
       "      <td>660.594204</td>\n",
       "      <td>group member usd woman group woman school</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>68</td>\n",
       "      <td>133</td>\n",
       "      <td>124.158622</td>\n",
       "      <td>fee school fee pay school expand expand business</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>383</td>\n",
       "      <td>489</td>\n",
       "      <td>402.629142</td>\n",
       "      <td>cow dairy milk dairy cow group farming</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>699</td>\n",
       "      <td>800</td>\n",
       "      <td>655.510271</td>\n",
       "      <td>group life small member hope dominican</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>311</td>\n",
       "      <td>383</td>\n",
       "      <td>328.322045</td>\n",
       "      <td>stock primary school able primary school increase</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>781</td>\n",
       "      <td>851</td>\n",
       "      <td>712.446289</td>\n",
       "      <td>school requesting requesting loan family buy lack</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6</td>\n",
       "      <td>137</td>\n",
       "      <td>181</td>\n",
       "      <td>169.067136</td>\n",
       "      <td>expand expand business father family main living</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>79</td>\n",
       "      <td>70.703796</td>\n",
       "      <td>farming maize land farmer farm bean</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>8</td>\n",
       "      <td>334</td>\n",
       "      <td>415</td>\n",
       "      <td>354.291304</td>\n",
       "      <td>santo domingo santo domingo store community husband</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>9</td>\n",
       "      <td>166</td>\n",
       "      <td>227</td>\n",
       "      <td>202.459533</td>\n",
       "      <td>kenya service community yearold applied expand</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>10</td>\n",
       "      <td>612</td>\n",
       "      <td>695</td>\n",
       "      <td>582.433888</td>\n",
       "      <td>product home store first new second</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "      <td>33</td>\n",
       "      <td>70.927938</td>\n",
       "      <td>source source income income church fruit farming</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>12</td>\n",
       "      <td>278</td>\n",
       "      <td>342</td>\n",
       "      <td>304.118714</td>\n",
       "      <td>nairobi also care take care take month</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>56.637560</td>\n",
       "      <td>applied every day secondary small business church material</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>14</td>\n",
       "      <td>425</td>\n",
       "      <td>483</td>\n",
       "      <td>421.169487</td>\n",
       "      <td>goal store house merchandise cement made</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>15</td>\n",
       "      <td>129</td>\n",
       "      <td>195</td>\n",
       "      <td>169.341325</td>\n",
       "      <td>rice land farm hope invest life</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>16</td>\n",
       "      <td>657</td>\n",
       "      <td>730</td>\n",
       "      <td>627.219731</td>\n",
       "      <td>work pm home help house husband</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>17</td>\n",
       "      <td>36</td>\n",
       "      <td>80</td>\n",
       "      <td>95.606818</td>\n",
       "      <td>feed educate year ago ago saving little</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>56.637560</td>\n",
       "      <td>applied every day secondary small business church material</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>19</td>\n",
       "      <td>802</td>\n",
       "      <td>858</td>\n",
       "      <td>737.724637</td>\n",
       "      <td>entrepreneur community clothing woman explains family</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    TopicID  Support50  Support10      Weight                                                       Terms\n",
       "0         0        700        789  660.594204                   group member usd woman group woman school\n",
       "1         1         68        133  124.158622            fee school fee pay school expand expand business\n",
       "2         2        383        489  402.629142                      cow dairy milk dairy cow group farming\n",
       "3         3        699        800  655.510271                      group life small member hope dominican\n",
       "4         4        311        383  328.322045           stock primary school able primary school increase\n",
       "5         5        781        851  712.446289           school requesting requesting loan family buy lack\n",
       "6         6        137        181  169.067136            expand expand business father family main living\n",
       "7         7          1         79   70.703796                         farming maize land farmer farm bean\n",
       "8         8        334        415  354.291304         santo domingo santo domingo store community husband\n",
       "9         9        166        227  202.459533              kenya service community yearold applied expand\n",
       "10       10        612        695  582.433888                         product home store first new second\n",
       "11       11          0         33   70.927938            source source income income church fruit farming\n",
       "12       12        278        342  304.118714                      nairobi also care take care take month\n",
       "13       13          0          0   56.637560  applied every day secondary small business church material\n",
       "14       14        425        483  421.169487                    goal store house merchandise cement made\n",
       "15       15        129        195  169.341325                             rice land farm hope invest life\n",
       "16       16        657        730  627.219731                             work pm home help house husband\n",
       "17       17         36         80   95.606818                     feed educate year ago ago saving little\n",
       "18       18          0          0   56.637560  applied every day secondary small business church material\n",
       "19       19        802        858  737.724637       entrepreneur community clothing woman explains family"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.set_option('display.max_colwidth', 100)\n",
    "pd.set_option('display.max_columns', 0)\n",
    "topic_summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyLDAvis.sklearn\n",
    " \n",
    "pyLDAvis.enable_notebook()\n",
    "% time pyLDAvis.sklearn.prepare(lda_model, dtm, vectorizer, mds=\"tsne\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}