instate/models/KNN_cosine_distance_simple_avg_modal_state.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "360e992a",
   "metadata": {},
   "source": [
    "### Predict State of Residence of Unseen Last Name Using KNN (Cosine Distance and Levenshtein)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5b19ec34",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import collections\n",
    "import Levenshtein as lv\n",
    "import multiprocessing as mp\n",
    "from tqdm import tqdm\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer                                                             \n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics.pairwise import cosine_similarity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "6b3d6963",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_dat = pd.read_csv(\"../../data/instate_processed_clean.csv.gz\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "40f585fc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(421190808, 3)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_dat.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7a8b0e71",
   "metadata": {},
   "outputs": [],
   "source": [
    "gdf = all_dat.groupby(['last_name','state'], as_index = False)['state'].agg(['count'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "28ec0579",
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating a pivot table so that each name has a count of the # of states with that last name\n",
    "gdf = gdf.pivot_table(values = 'count', columns = 'state', index = 'last_name')\n",
    "\n",
    "# Converting NaN to zeros since that means there is no one that lives in that state with that last name\n",
    "gdf = gdf.fillna(0)\n",
    "\n",
    "# Getting the totals of each last name\n",
    "gdf['total_n'] = gdf.sum(axis = 1)\n",
    "gdf.reset_index(inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "b8ff7054",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 27.4 s, sys: 515 ms, total: 28 s\n",
      "Wall time: 27.7 s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>state</th>\n",
       "      <th>last_name</th>\n",
       "      <th>andaman</th>\n",
       "      <th>andhra</th>\n",
       "      <th>arunachal</th>\n",
       "      <th>assam</th>\n",
       "      <th>bihar</th>\n",
       "      <th>chandigarh</th>\n",
       "      <th>dadra</th>\n",
       "      <th>daman</th>\n",
       "      <th>delhi</th>\n",
       "      <th>...</th>\n",
       "      <th>odi</th>\n",
       "      <th>puducherry</th>\n",
       "      <th>punjab</th>\n",
       "      <th>rajasthan</th>\n",
       "      <th>sikkim</th>\n",
       "      <th>tel</th>\n",
       "      <th>tripura</th>\n",
       "      <th>up</th>\n",
       "      <th>utt</th>\n",
       "      <th>total_n</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>aaa</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000136</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.017176</td>\n",
       "      <td>...</td>\n",
       "      <td>0.054253</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001772</td>\n",
       "      <td>0.01813</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.859733</td>\n",
       "      <td>0.016903</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>aaaa</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.003175</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.009524</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.942857</td>\n",
       "      <td>0.028571</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>aaaaa</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>aaaaaa</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.996815</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>aaaabaaraav</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1140998</th>\n",
       "      <td>ൟadiaaa</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1140999</th>\n",
       "      <td>ൟaraaia</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1141000</th>\n",
       "      <td>ൟasadeia</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1141001</th>\n",
       "      <td>ൟithi</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1141002</th>\n",
       "      <td>ൟunusa</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1141003 rows × 33 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "state      last_name  andaman    andhra  arunachal     assam  bihar  \\\n",
       "0                aaa      0.0  0.000136        0.0  0.000000    0.0   \n",
       "1               aaaa      0.0  0.000000        0.0  0.003175    0.0   \n",
       "2              aaaaa      0.0  0.000000        0.0  0.000000    0.0   \n",
       "3             aaaaaa      0.0  0.000000        0.0  0.000000    0.0   \n",
       "4        aaaabaaraav      0.0  0.000000        0.0  0.000000    0.0   \n",
       "...              ...      ...       ...        ...       ...    ...   \n",
       "1140998      ൟadiaaa      0.0  0.000000        0.0  0.000000    0.0   \n",
       "1140999      ൟaraaia      0.0  0.000000        0.0  0.000000    0.0   \n",
       "1141000     ൟasadeia      0.0  0.000000        0.0  0.000000    0.0   \n",
       "1141001        ൟithi      0.0  0.000000        0.0  0.000000    0.0   \n",
       "1141002       ൟunusa      0.0  0.000000        0.0  0.000000    0.0   \n",
       "\n",
       "state    chandigarh  dadra  daman     delhi  ...       odi  puducherry  \\\n",
       "0               0.0    0.0    0.0  0.017176  ...  0.054253         0.0   \n",
       "1               0.0    0.0    0.0  0.000000  ...  0.009524         0.0   \n",
       "2               0.0    0.0    0.0  0.000000  ...  0.000000         0.0   \n",
       "3               0.0    0.0    0.0  0.000000  ...  0.000000         0.0   \n",
       "4               0.0    0.0    0.0  0.000000  ...  0.000000         0.0   \n",
       "...             ...    ...    ...       ...  ...       ...         ...   \n",
       "1140998         0.0    0.0    0.0  0.000000  ...  1.000000         0.0   \n",
       "1140999         0.0    0.0    0.0  0.000000  ...  1.000000         0.0   \n",
       "1141000         0.0    0.0    0.0  0.000000  ...  1.000000         0.0   \n",
       "1141001         0.0    0.0    0.0  0.000000  ...  1.000000         0.0   \n",
       "1141002         0.0    0.0    0.0  0.000000  ...  1.000000         0.0   \n",
       "\n",
       "state    punjab  rajasthan    sikkim      tel  tripura        up       utt  \\\n",
       "0           0.0        0.0  0.001772  0.01813      0.0  0.859733  0.016903   \n",
       "1           0.0        0.0  0.000000  0.00000      0.0  0.942857  0.028571   \n",
       "2           0.0        0.0  0.000000  0.00000      0.0  1.000000  0.000000   \n",
       "3           0.0        0.0  0.000000  0.00000      0.0  0.996815  0.000000   \n",
       "4           0.0        0.0  0.000000  0.00000      0.0  0.000000  0.000000   \n",
       "...         ...        ...       ...      ...      ...       ...       ...   \n",
       "1140998     0.0        0.0  0.000000  0.00000      0.0  0.000000  0.000000   \n",
       "1140999     0.0        0.0  0.000000  0.00000      0.0  0.000000  0.000000   \n",
       "1141000     0.0        0.0  0.000000  0.00000      0.0  0.000000  0.000000   \n",
       "1141001     0.0        0.0  0.000000  0.00000      0.0  0.000000  0.000000   \n",
       "1141002     0.0        0.0  0.000000  0.00000      0.0  0.000000  0.000000   \n",
       "\n",
       "state    total_n  \n",
       "0            1.0  \n",
       "1            1.0  \n",
       "2            1.0  \n",
       "3            1.0  \n",
       "4            1.0  \n",
       "...          ...  \n",
       "1140998      1.0  \n",
       "1140999      1.0  \n",
       "1141000      1.0  \n",
       "1141001      1.0  \n",
       "1141002      1.0  \n",
       "\n",
       "[1141003 rows x 33 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "# Calculate the proportion of people with a particular last name that live in various states\n",
    "gdf.iloc[:, 1:] = gdf.iloc[:, 1:].div(gdf.total_n, axis = 0)\n",
    "gdf.to_csv(\"../../data/instate_unique_ln_state_prop_v1.csv.gz\",\n",
    "           compression='gzip')\n",
    "gdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "1b323eb4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    kerala\n",
       "1        up\n",
       "2        up\n",
       "3        up\n",
       "4        up\n",
       "Name: modal_state, dtype: object"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gdf['modal_state'] = gdf[gdf.columns[1:-1]].idxmax(axis=1)\n",
    "gdf['modal_state'].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "5a1199f3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['last_name', 'andaman', 'andhra', 'arunachal', 'assam', 'bihar',\n",
       "       'chandigarh', 'dadra', 'daman', 'delhi', 'goa', 'guj', 'har', 'jha',\n",
       "       'jk', 'kar', 'kerala', 'maharashtra', 'manipur', 'meghalaya', 'mizoram',\n",
       "       'mp', 'nagaland', 'odi', 'puducherry', 'punjab', 'rajasthan', 'sikkim',\n",
       "       'tel', 'tripura', 'up', 'utt', 'total_n', 'modal_state'],\n",
       "      dtype='object', name='state')"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gdf.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "16cdb34a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1128299, 34)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "proto_df = gdf.groupby('modal_state', group_keys=False).apply(lambda x: x.sample(frac=1, random_state=10))\n",
    "proto_df.reset_index(inplace=True)\n",
    "proto_df.drop('index', axis=1, inplace=True)\n",
    "proto_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "20adab8d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "maharashtra    341179\n",
       "andhra         229327\n",
       "guj            136235\n",
       "kar             62541\n",
       "odi             49085\n",
       "bihar           37957\n",
       "up              36526\n",
       "delhi           35801\n",
       "kerala          34938\n",
       "mp              33698\n",
       "manipur         25400\n",
       "jha             15991\n",
       "assam           11566\n",
       "goa              9783\n",
       "mizoram          9296\n",
       "rajasthan        9106\n",
       "arunachal        7000\n",
       "tel              5944\n",
       "nagaland         5447\n",
       "har              4703\n",
       "punjab           4672\n",
       "utt              3835\n",
       "dadra            3312\n",
       "meghalaya        3189\n",
       "tripura          3075\n",
       "puducherry       2421\n",
       "sikkim           1755\n",
       "daman            1753\n",
       "jk               1094\n",
       "andaman           896\n",
       "chandigarh        774\n",
       "Name: modal_state, dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "proto_df.modal_state.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "a56d1ea7",
   "metadata": {},
   "outputs": [],
   "source": [
    "#test_agg = test.groupby(['last_name'])['state'].value_counts(normalize=True).unstack(-1).fillna(0).reset_index()\n",
    "#top_three = test_agg.set_index('last_name')\n",
    "#top_three = pd.DataFrame(top_three.columns.values[np.argsort(-top_three.values, axis=1)[:, :3]], \n",
    "#                  index=top_three.index,\n",
    "#                 columns = ['1st Max','2nd Max','3rd Max']).reset_index()\n",
    "#top_three.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "4d472f73",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 6.04 s, sys: 68.2 ms, total: 6.11 s\n",
      "Wall time: 6.15 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# build n-gram list\n",
    "NGRAMS = 2\n",
    "vect = CountVectorizer(analyzer='char', max_df=0.5, min_df=.005, ngram_range=(NGRAMS, NGRAMS), lowercase=False) \n",
    "tfidf_transformer = TfidfTransformer()\n",
    "\n",
    "a = vect.fit_transform(proto_df.last_name) \n",
    "tfidf = tfidf_transformer.fit_transform(a)\n",
    "\n",
    "vocab = vect.vocabulary_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "1ecc7af2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['ha', 'at', 'va', 'bi', 'in', 'nd', 'du', 'uj', 'ja']\n",
      "num_words = 195\n"
     ]
    }
   ],
   "source": [
    "words = []\n",
    "for b in vocab:\n",
    "    c = vocab[b]\n",
    "    words.append((a[:, c].sum(), b))\n",
    "\n",
    "words_list = [w[1] for w in words]\n",
    "print(words_list[1:10])\n",
    "num_words = len(words_list)\n",
    "print(\"num_words = %d\" % num_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "5a584e18",
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_ngrams(text, n):\n",
    "    a = zip(*[text[i:] for i in range(n)])\n",
    "    wi = []\n",
    "    for i in a:\n",
    "        w = ''.join(i)\n",
    "        try:\n",
    "            idx = words_list.index(w)\n",
    "        except:\n",
    "            idx = 0\n",
    "        wi.append(idx)\n",
    "    return wi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "03107066",
   "metadata": {},
   "outputs": [],
   "source": [
    "# build tf-idf vectors\n",
    "proto_df['tfidf_index'] = proto_df.index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "b49d193a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set size: (1018289, 36)\n",
      "Validation set size: (53595, 36)\n",
      "Test set size: (56415, 36)\n"
     ]
    }
   ],
   "source": [
    "train_df, test_df = train_test_split(proto_df, test_size=.05, random_state= 10)\n",
    "train_df, valid_df = train_test_split(train_df, test_size=.05, random_state= 10)\n",
    "\n",
    "train_df.reset_index(inplace=True)\n",
    "valid_df.reset_index(inplace=True)\n",
    "test_df.reset_index(inplace=True)\n",
    "\n",
    "print('Training set size: {}'.format(train_df.shape))\n",
    "print('Validation set size: {}'.format(valid_df.shape))\n",
    "print('Test set size: {}'.format(test_df.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "77a242f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_cosine_state(arg):\n",
    "    # reading the tuple passed on by the calling function\n",
    "    idx, row_data, test_df, corpus_df, corp_vector, k = arg\n",
    "    \n",
    "    # resizing the tf-idf (1, m) & corpus vectors to be (n, m)\n",
    "    #  n = number of samples\n",
    "    #  m = number of dimentions\n",
    "    orig_vector = tfidf[row_data['tfidf_index']].reshape(1, -1)\n",
    "\n",
    "    # calculating the cosine similarity beteween the name vector\n",
    "    #   and the corpus vectors.  Then filtering for only values\n",
    "    #   that are greater that what was passed on\n",
    "    cossim = cosine_similarity(orig_vector, corp_vector)\n",
    "        \n",
    "    # Order by cosine distance and pick top k\n",
    "    cossim_df = corpus_df.iloc[np.flip(cossim.flatten().argsort())[:k]]\n",
    "    \n",
    "    pred_state = states[cossim_df[states].mean().argmax()]\n",
    "    test_df.loc[idx, 'pred_state'] = pred_state\n",
    "    return pred_state\n",
    "\n",
    "def check_cosine_k(test_df, corpus_df, k):\n",
    "    results = []\n",
    "\n",
    "    num_cpu = mp.cpu_count() \n",
    "    pool = mp.pool.ThreadPool(processes=8)\n",
    "\n",
    "    corp_vector = tfidf[corpus_df['tfidf_index']]\n",
    "\n",
    "    # for idx, row in tqdm(test_df.iterrows()):\n",
    "    r = pool.map(predict_cosine_state, [(idx, row, test_df, corpus_df, corp_vector, k)\n",
    "                                for idx, row in test_df.iterrows()])\n",
    "    results.append(r)\n",
    "\n",
    "    pool.close()\n",
    "    pool.join()\n",
    "\n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "5fad3d34",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['andaman', 'andhra', 'arunachal', 'assam', 'bihar', 'chandigarh',\n",
       "       'dadra', 'daman', 'delhi', 'goa', 'guj', 'har', 'jha', 'jk', 'kar',\n",
       "       'kerala', 'maharashtra', 'manipur', 'meghalaya', 'mizoram', 'mp',\n",
       "       'nagaland', 'odi', 'puducherry', 'punjab', 'rajasthan', 'sikkim', 'tel',\n",
       "       'tripura', 'up', 'utt'],\n",
       "      dtype='object', name='state')"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "states = gdf.columns[1:-2]\n",
    "states"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "83dd928f",
   "metadata": {},
   "outputs": [],
   "source": [
    "k_metrics = {\n",
    "    3:0,\n",
    "    5:0,\n",
    "    10:0\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "fb1a4206",
   "metadata": {},
   "outputs": [],
   "source": [
    "true_list = []\n",
    "for idx, row in valid_df.iterrows():\n",
    "    true_list.append(row['modal_state'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "f354ec20",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "for value of k: 3 \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "     andaman       0.04      0.02      0.03        47\n",
      "      andhra       0.72      0.83      0.77     10917\n",
      "   arunachal       0.28      0.24      0.26       339\n",
      "       assam       0.33      0.29      0.31       572\n",
      "       bihar       0.41      0.30      0.34      1781\n",
      "  chandigarh       0.00      0.00      0.00        41\n",
      "       dadra       0.41      0.21      0.27       150\n",
      "       daman       0.16      0.09      0.12        85\n",
      "       delhi       0.41      0.44      0.42      1695\n",
      "         goa       0.40      0.29      0.33       470\n",
      "         guj       0.80      0.75      0.77      6479\n",
      "         har       0.18      0.06      0.09       233\n",
      "         jha       0.37      0.28      0.32       784\n",
      "          jk       0.26      0.22      0.23        51\n",
      "         kar       0.66      0.60      0.63      2974\n",
      "      kerala       0.45      0.41      0.43      1653\n",
      " maharashtra       0.69      0.77      0.73     16311\n",
      "     manipur       0.62      0.52      0.56      1222\n",
      "   meghalaya       0.38      0.28      0.32       149\n",
      "     mizoram       0.70      0.64      0.67       423\n",
      "          mp       0.39      0.30      0.34      1545\n",
      "    nagaland       0.38      0.29      0.33       254\n",
      "         odi       0.50      0.50      0.50      2262\n",
      "  puducherry       0.40      0.12      0.18       118\n",
      "      punjab       0.26      0.14      0.18       219\n",
      "   rajasthan       0.23      0.14      0.17       408\n",
      "      sikkim       0.51      0.36      0.42        92\n",
      "         tel       0.25      0.19      0.22       283\n",
      "     tripura       0.65      0.42      0.51       159\n",
      "          up       0.33      0.26      0.29      1693\n",
      "         utt       0.38      0.21      0.27       186\n",
      "\n",
      "    accuracy                           0.64     53595\n",
      "   macro avg       0.40      0.33      0.36     53595\n",
      "weighted avg       0.62      0.64      0.63     53595\n",
      "\n",
      "for value of k: 5 \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "     andaman       0.09      0.02      0.03        47\n",
      "      andhra       0.74      0.83      0.78     10917\n",
      "   arunachal       0.36      0.24      0.29       339\n",
      "       assam       0.38      0.25      0.30       572\n",
      "       bihar       0.44      0.28      0.34      1781\n",
      "  chandigarh       0.00      0.00      0.00        41\n",
      "       dadra       0.57      0.17      0.27       150\n",
      "       daman       0.53      0.11      0.18        85\n",
      "       delhi       0.44      0.44      0.44      1695\n",
      "         goa       0.54      0.25      0.34       470\n",
      "         guj       0.81      0.74      0.78      6479\n",
      "         har       0.15      0.03      0.06       233\n",
      "         jha       0.38      0.24      0.30       784\n",
      "          jk       0.40      0.16      0.23        51\n",
      "         kar       0.71      0.61      0.66      2974\n",
      "      kerala       0.45      0.40      0.42      1653\n",
      " maharashtra       0.67      0.82      0.74     16311\n",
      "     manipur       0.65      0.52      0.58      1222\n",
      "   meghalaya       0.46      0.28      0.34       149\n",
      "     mizoram       0.72      0.66      0.69       423\n",
      "          mp       0.43      0.34      0.38      1545\n",
      "    nagaland       0.45      0.29      0.35       254\n",
      "         odi       0.52      0.50      0.51      2262\n",
      "  puducherry       0.40      0.05      0.09       118\n",
      "      punjab       0.29      0.14      0.19       219\n",
      "   rajasthan       0.22      0.11      0.14       408\n",
      "      sikkim       0.65      0.38      0.48        92\n",
      "         tel       0.25      0.18      0.21       283\n",
      "     tripura       0.72      0.45      0.55       159\n",
      "          up       0.34      0.26      0.30      1693\n",
      "         utt       0.45      0.20      0.28       186\n",
      "\n",
      "    accuracy                           0.65     53595\n",
      "   macro avg       0.46      0.32      0.36     53595\n",
      "weighted avg       0.63      0.65      0.64     53595\n",
      "\n",
      "for value of k: 10 \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "     andaman       0.17      0.02      0.04        47\n",
      "      andhra       0.73      0.84      0.78     10917\n",
      "   arunachal       0.40      0.17      0.24       339\n",
      "       assam       0.46      0.22      0.30       572\n",
      "       bihar       0.45      0.23      0.30      1781\n",
      "  chandigarh       0.00      0.00      0.00        41\n",
      "       dadra       0.70      0.13      0.21       150\n",
      "       daman       1.00      0.05      0.09        85\n",
      "       delhi       0.46      0.44      0.45      1695\n",
      "         goa       0.65      0.20      0.31       470\n",
      "         guj       0.81      0.74      0.77      6479\n",
      "         har       0.24      0.03      0.05       233\n",
      "         jha       0.45      0.21      0.29       784\n",
      "          jk       0.60      0.12      0.20        51\n",
      "         kar       0.72      0.57      0.64      2974\n",
      "      kerala       0.45      0.39      0.42      1653\n",
      " maharashtra       0.64      0.85      0.73     16311\n",
      "     manipur       0.69      0.53      0.60      1222\n",
      "   meghalaya       0.47      0.21      0.29       149\n",
      "     mizoram       0.76      0.66      0.71       423\n",
      "          mp       0.44      0.31      0.36      1545\n",
      "    nagaland       0.52      0.29      0.37       254\n",
      "         odi       0.55      0.48      0.51      2262\n",
      "  puducherry       0.50      0.06      0.11       118\n",
      "      punjab       0.32      0.09      0.14       219\n",
      "   rajasthan       0.29      0.09      0.13       408\n",
      "      sikkim       0.59      0.29      0.39        92\n",
      "         tel       0.26      0.19      0.22       283\n",
      "     tripura       0.80      0.42      0.55       159\n",
      "          up       0.35      0.23      0.28      1693\n",
      "         utt       0.37      0.12      0.18       186\n",
      "\n",
      "    accuracy                           0.65     53595\n",
      "   macro avg       0.51      0.30      0.34     53595\n",
      "weighted avg       0.64      0.65      0.63     53595\n",
      "\n",
      "CPU times: user 11h 58min 26s, sys: 32min 50s, total: 12h 31min 16s\n",
      "Wall time: 11h 31min 28s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "for value, key in enumerate (k_metrics):\n",
    "    #print ('{} -- {}'.format(key, value))\n",
    "    result = check_cosine_k(valid_df, train_df, key)\n",
    "    \n",
    "    pred_list = np.array(result).reshape(-1)\n",
    "    pred_list = pred_list.tolist()\n",
    "    \n",
    "    true_list = pd.Series(true_list).tolist() #.replace(replacement).to_list()\n",
    "    pred_list = pd.Series(pred_list).tolist() #.replace(replacement).to_list()\n",
    "    \n",
    "    value = classification_report(true_list, pred_list, zero_division = 0)\n",
    "    \n",
    "    print ('for value of k: {} \\n{}'.format(key, value))\n",
    "    k_metrics[key] = value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "52735b28",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 4h 10min 21s, sys: 11min 29s, total: 4h 21min 51s\n",
      "Wall time: 1h 43min 34s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "result = check_cosine_k(test_df, train_df, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "d28c4412",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "     andaman       0.09      0.02      0.03        50\n",
      "      andhra       0.73      0.83      0.78     11249\n",
      "   arunachal       0.34      0.21      0.26       348\n",
      "       assam       0.46      0.31      0.37       553\n",
      "       bihar       0.44      0.28      0.34      1880\n",
      "  chandigarh       0.00      0.00      0.00        47\n",
      "       dadra       0.60      0.21      0.31       145\n",
      "       daman       0.38      0.10      0.15        93\n",
      "       delhi       0.45      0.48      0.46      1799\n",
      "         goa       0.55      0.22      0.31       541\n",
      "         guj       0.81      0.74      0.78      6967\n",
      "         har       0.21      0.05      0.08       248\n",
      "         jha       0.38      0.26      0.31       766\n",
      "          jk       0.18      0.07      0.10        46\n",
      "         kar       0.69      0.59      0.64      3150\n",
      "      kerala       0.48      0.42      0.45      1712\n",
      " maharashtra       0.67      0.82      0.74     17104\n",
      "     manipur       0.65      0.51      0.57      1260\n",
      "   meghalaya       0.50      0.25      0.33       167\n",
      "     mizoram       0.74      0.62      0.67       470\n",
      "          mp       0.44      0.34      0.38      1646\n",
      "    nagaland       0.44      0.31      0.36       266\n",
      "         odi       0.53      0.51      0.52      2501\n",
      "  puducherry       0.44      0.09      0.15       123\n",
      "      punjab       0.30      0.15      0.20       218\n",
      "   rajasthan       0.31      0.13      0.18       485\n",
      "      sikkim       0.57      0.32      0.41        92\n",
      "         tel       0.30      0.24      0.26       304\n",
      "     tripura       0.66      0.51      0.57       144\n",
      "          up       0.36      0.27      0.31      1840\n",
      "         utt       0.44      0.21      0.28       201\n",
      "\n",
      "    accuracy                           0.65     56415\n",
      "   macro avg       0.46      0.32      0.36     56415\n",
      "weighted avg       0.63      0.65      0.64     56415\n",
      "\n"
     ]
    }
   ],
   "source": [
    "pred_list = np.array(result).reshape(-1)\n",
    "pred_list = pred_list.tolist()\n",
    "\n",
    "true_list = []\n",
    "for idx, row in test_df.iterrows():\n",
    "    true_list.append(row['modal_state'])\n",
    "\n",
    "true_list = pd.Series(true_list).tolist() #.replace(replacement).to_list()\n",
    "pred_list = pd.Series(pred_list).tolist() #.replace(replacement).to_list()\n",
    "\n",
    "value = classification_report(true_list, pred_list, zero_division = 0)\n",
    "\n",
    "print (value)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}