ethnicolr/models/ethnicolr_keras_lstm_fl_voter_name.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-12-21 15:33:07.636536: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory\n",
      "2021-12-21 15:33:07.636592: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.5.0\n",
      "2.5.2\n"
     ]
    }
   ],
   "source": [
    "import keras\n",
    "import tensorflow as tf\n",
    "print(keras.__version__)\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name_last</th>\n",
       "      <th>name_first</th>\n",
       "      <th>race</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5577783</th>\n",
       "      <td>Lidros</td>\n",
       "      <td>Johan</td>\n",
       "      <td>nh_white</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11025099</th>\n",
       "      <td>Dionne</td>\n",
       "      <td>John</td>\n",
       "      <td>nh_white</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9175381</th>\n",
       "      <td>Serrano Ortiz</td>\n",
       "      <td>Hilda</td>\n",
       "      <td>hispanic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>511481</th>\n",
       "      <td>Hampton</td>\n",
       "      <td>Tracy</td>\n",
       "      <td>nh_white</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>336833</th>\n",
       "      <td>Mitchell</td>\n",
       "      <td>Michael</td>\n",
       "      <td>nh_white</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7765170</th>\n",
       "      <td>Osipov</td>\n",
       "      <td>Konstantin</td>\n",
       "      <td>nh_white</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8437763</th>\n",
       "      <td>Ammerman</td>\n",
       "      <td>Daniel</td>\n",
       "      <td>nh_white</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>816193</th>\n",
       "      <td>Goldberg</td>\n",
       "      <td>Randolph</td>\n",
       "      <td>nh_white</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13135601</th>\n",
       "      <td>Carrick</td>\n",
       "      <td>Donald</td>\n",
       "      <td>nh_white</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1545274</th>\n",
       "      <td>Mc Minn</td>\n",
       "      <td>Beverly</td>\n",
       "      <td>nh_white</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000000 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              name_last  name_first      race\n",
       "5577783          Lidros       Johan  nh_white\n",
       "11025099         Dionne        John  nh_white\n",
       "9175381   Serrano Ortiz       Hilda  hispanic\n",
       "511481          Hampton       Tracy  nh_white\n",
       "336833         Mitchell     Michael  nh_white\n",
       "...                 ...         ...       ...\n",
       "7765170          Osipov  Konstantin  nh_white\n",
       "8437763        Ammerman      Daniel  nh_white\n",
       "816193         Goldberg    Randolph  nh_white\n",
       "13135601        Carrick      Donald  nh_white\n",
       "1545274         Mc Minn     Beverly  nh_white\n",
       "\n",
       "[1000000 rows x 3 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import classification_report,confusion_matrix\n",
    "\n",
    "NGRAMS = 2\n",
    "SAMPLE = 1000000\n",
    "EPOCHS = 15\n",
    "\n",
    "# Florida voter\n",
    "df = pd.read_csv('/opt/data/fl_voterreg/fl_reg_name_race.csv.gz')\n",
    "df.dropna(subset=['name_first', 'name_last'], inplace=True)\n",
    "sdf = df[df.race.isin(['multi_racial', 'native_indian', 'other', 'unknown']) == False].sample(SAMPLE, random_state=21)\n",
    "del df\n",
    "\n",
    "# Additional features\n",
    "sdf['name_first'] = sdf.name_first.str.title()\n",
    "sdf['name_last'] = sdf.name_last.str.title()\n",
    "\n",
    "sdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name_first</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>race</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>asian</th>\n",
       "      <td>19382</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hispanic</th>\n",
       "      <td>167274</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nh_black</th>\n",
       "      <td>141448</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nh_white</th>\n",
       "      <td>671896</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          name_first\n",
       "race                \n",
       "asian          19382\n",
       "hispanic      167274\n",
       "nh_black      141448\n",
       "nh_white      671896"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rdf = sdf.groupby('race').agg({'name_first': 'count'})\n",
    "rdf.to_csv('./fl_voter_reg/lstm/fl_name_race.csv', columns=[])\n",
    "rdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name_last</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>race</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>asian</th>\n",
       "      <td>9362</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hispanic</th>\n",
       "      <td>41246</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nh_black</th>\n",
       "      <td>22837</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nh_white</th>\n",
       "      <td>145560</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          name_last\n",
       "race               \n",
       "asian          9362\n",
       "hispanic      41246\n",
       "nh_black      22837\n",
       "nh_white     145560"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sdf.groupby('race').agg({'name_last': 'nunique'})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocessing the input data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num_words = 1260\n",
      "Max feature len = 41, Avg. feature len = 12\n"
     ]
    }
   ],
   "source": [
    "# concat last name and first name\n",
    "sdf['name_last_name_first'] = sdf['name_last'] + ' ' + sdf['name_first']\n",
    "\n",
    "# build n-gram list\n",
    "vect = CountVectorizer(analyzer='char', max_df=0.3, min_df=3, ngram_range=(NGRAMS, NGRAMS), lowercase=False) \n",
    "a = vect.fit_transform(sdf.name_last_name_first)\n",
    "vocab = vect.vocabulary_\n",
    "\n",
    "# sort n-gram by freq (highest -> lowest)\n",
    "words = []\n",
    "for b in vocab:\n",
    "    c = vocab[b]\n",
    "    #print(b, c, a[:, c].sum())\n",
    "    words.append((a[:, c].sum(), b))\n",
    "    #break\n",
    "words = sorted(words, reverse=True)\n",
    "words_list = ['UNK']\n",
    "words_list.extend([w[1] for w in words])\n",
    "num_words = len(words_list)\n",
    "print(\"num_words = %d\" % num_words)\n",
    "\n",
    "\n",
    "def find_ngrams(text, n):\n",
    "    a = zip(*[text[i:] for i in range(n)])\n",
    "    wi = []\n",
    "    for i in a:\n",
    "        w = ''.join(i)\n",
    "        try:\n",
    "            idx = words_list.index(w)\n",
    "        except:\n",
    "            idx = 0\n",
    "        wi.append(idx)\n",
    "    return wi\n",
    "\n",
    "# build X from index of n-gram sequence\n",
    "X = np.array(sdf.name_last_name_first.apply(lambda c: find_ngrams(c, NGRAMS)))\n",
    "\n",
    "# check max/avg feature\n",
    "X_len = []\n",
    "for x in X:\n",
    "    X_len.append(len(x))\n",
    "\n",
    "max_feature_len = max(X_len)\n",
    "avg_feature_len = int(np.mean(X_len))\n",
    "\n",
    "print(\"Max feature len = %d, Avg. feature len = %d\" % (max_feature_len, avg_feature_len))\n",
    "y = np.array(sdf.race.astype('category').cat.codes)\n",
    "\n",
    "# Split train and test dataset\n",
    "X_train,  X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=21, stratify=y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train a LSTM model\n",
    "\n",
    "ref: http://machinelearningmastery.com/sequence-classification-lstm-recurrent-neural-networks-python-keras/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "800000 train sequences\n",
      "200000 test sequences\n",
      "Pad sequences (samples x time)\n",
      "X_train shape: (800000, 25)\n",
      "X_test shape: (200000, 25)\n",
      "4 classes\n",
      "Convert class vector to binary class matrix (for use with categorical_crossentropy)\n",
      "y_train shape: (800000, 4)\n",
      "y_test shape: (200000, 4)\n"
     ]
    }
   ],
   "source": [
    "'''The dataset is actually too small for LSTM to be of any advantage\n",
    "compared to simpler, much faster methods such as TF-IDF + LogReg.\n",
    "Notes:\n",
    "\n",
    "- RNNs are tricky. Choice of batch size is important,\n",
    "choice of loss and optimizer is critical, etc.\n",
    "Some configurations won't converge.\n",
    "\n",
    "- LSTM loss decrease patterns during training can be quite different\n",
    "from what you see with CNNs/MLPs/etc.\n",
    "'''\n",
    "from keras.preprocessing import sequence\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Embedding, Dropout, Activation\n",
    "from keras.layers import LSTM\n",
    "from keras.layers.convolutional import Conv1D\n",
    "from keras.layers.convolutional import MaxPooling1D\n",
    "from keras.models import load_model\n",
    "\n",
    "max_features = num_words # 20000\n",
    "feature_len = 25 # avg_feature_len # cut texts after this number of words (among top max_features most common words)\n",
    "batch_size = 32\n",
    "\n",
    "print(len(X_train), 'train sequences')\n",
    "print(len(X_test), 'test sequences')\n",
    "\n",
    "print('Pad sequences (samples x time)')\n",
    "X_train = sequence.pad_sequences(X_train, maxlen=feature_len)\n",
    "X_test = sequence.pad_sequences(X_test, maxlen=feature_len)\n",
    "print('X_train shape:', X_train.shape)\n",
    "print('X_test shape:', X_test.shape)\n",
    "\n",
    "num_classes = np.max(y_train) + 1\n",
    "print(num_classes, 'classes')\n",
    "\n",
    "print('Convert class vector to binary class matrix '\n",
    "      '(for use with categorical_crossentropy)')\n",
    "y_train = tf.keras.utils.to_categorical(y_train, num_classes)\n",
    "y_test = tf.keras.utils.to_categorical(y_test, num_classes)\n",
    "print('y_train shape:', y_train.shape)\n",
    "print('y_test shape:', y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Build model...\n",
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding (Embedding)        (None, 25, 32)            40320     \n",
      "_________________________________________________________________\n",
      "lstm (LSTM)                  (None, 128)               82432     \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 4)                 516       \n",
      "=================================================================\n",
      "Total params: 123,268\n",
      "Trainable params: 123,268\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-12-21 15:36:04.374035: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory\n",
      "2021-12-21 15:36:04.374094: W tensorflow/stream_executor/cuda/cuda_driver.cc:326] failed call to cuInit: UNKNOWN ERROR (303)\n",
      "2021-12-21 15:36:04.374122: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:156] kernel driver does not appear to be running on this host (jupyter-server-2): /proc/driver/nvidia/version does not exist\n",
      "2021-12-21 15:36:04.374353: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
     ]
    }
   ],
   "source": [
    "print('Build model...')\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Embedding(num_words, 32, input_length=feature_len))\n",
    "model.add(LSTM(128, dropout=0.2, recurrent_dropout=0.2))\n",
    "model.add(Dense(num_classes, activation='softmax'))\n",
    "\n",
    "# try using different optimizers and different optimizer configs\n",
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer='adam',\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-12-21 15:36:04.839015: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:176] None of the MLIR Optimization Passes are enabled (registered 2)\n",
      "2021-12-21 15:36:04.839629: I tensorflow/core/platform/profile_utils/cpu_utils.cc:114] CPU Frequency: 2250000000 Hz\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/15\n",
      "22500/22500 [==============================] - 701s 30ms/step - loss: 0.6106 - accuracy: 0.7788 - val_loss: 0.5192 - val_accuracy: 0.8136\n",
      "Epoch 2/15\n",
      "22500/22500 [==============================] - 664s 30ms/step - loss: 0.5200 - accuracy: 0.8124 - val_loss: 0.4948 - val_accuracy: 0.8243\n",
      "Epoch 3/15\n",
      "22500/22500 [==============================] - 776s 34ms/step - loss: 0.4955 - accuracy: 0.8222 - val_loss: 0.4810 - val_accuracy: 0.8296\n",
      "Epoch 4/15\n",
      "22500/22500 [==============================] - 782s 35ms/step - loss: 0.4816 - accuracy: 0.8278 - val_loss: 0.4723 - val_accuracy: 0.8342\n",
      "Epoch 5/15\n",
      "22500/22500 [==============================] - 764s 34ms/step - loss: 0.4751 - accuracy: 0.8308 - val_loss: 0.4699 - val_accuracy: 0.8338\n",
      "Epoch 6/15\n",
      "22500/22500 [==============================] - 995s 44ms/step - loss: 0.4695 - accuracy: 0.8329 - val_loss: 0.4654 - val_accuracy: 0.8369\n",
      "Epoch 7/15\n",
      "22500/22500 [==============================] - 1149s 51ms/step - loss: 0.4655 - accuracy: 0.8348 - val_loss: 0.4613 - val_accuracy: 0.8375\n",
      "Epoch 8/15\n",
      "22500/22500 [==============================] - 1239s 55ms/step - loss: 0.4644 - accuracy: 0.8353 - val_loss: 0.4607 - val_accuracy: 0.8380\n",
      "Epoch 9/15\n",
      "22500/22500 [==============================] - 927s 41ms/step - loss: 0.4619 - accuracy: 0.8362 - val_loss: 0.4583 - val_accuracy: 0.8390\n",
      "Epoch 10/15\n",
      "22500/22500 [==============================] - 805s 36ms/step - loss: 0.4599 - accuracy: 0.8369 - val_loss: 0.4562 - val_accuracy: 0.8402\n",
      "Epoch 11/15\n",
      "22500/22500 [==============================] - 808s 36ms/step - loss: 0.4567 - accuracy: 0.8379 - val_loss: 0.4549 - val_accuracy: 0.8401\n",
      "Epoch 12/15\n",
      "22500/22500 [==============================] - 809s 36ms/step - loss: 0.4570 - accuracy: 0.8378 - val_loss: 0.4563 - val_accuracy: 0.8399\n",
      "Epoch 13/15\n",
      "22500/22500 [==============================] - 747s 33ms/step - loss: 0.4575 - accuracy: 0.8377 - val_loss: 0.4543 - val_accuracy: 0.8402\n",
      "Epoch 14/15\n",
      "22500/22500 [==============================] - 496s 22ms/step - loss: 0.4551 - accuracy: 0.8386 - val_loss: 0.4549 - val_accuracy: 0.8403\n",
      "Epoch 15/15\n",
      "22500/22500 [==============================] - 491s 22ms/step - loss: 0.4536 - accuracy: 0.8385 - val_loss: 0.4546 - val_accuracy: 0.8398\n",
      "6250/6250 [==============================] - 34s 6ms/step - loss: 0.4539 - accuracy: 0.8408\n",
      "Test score: 0.45394647121429443\n",
      "Test accuracy: 0.8408349752426147\n"
     ]
    }
   ],
   "source": [
    "print('Train...')\n",
    "model.fit(X_train, y_train, batch_size=batch_size, epochs=EPOCHS,\n",
    "          validation_split=0.1, verbose=1)\n",
    "score, acc = model.evaluate(X_test, y_test,\n",
    "                            batch_size=batch_size, verbose=1)\n",
    "print('Test score:', score)\n",
    "print('Test accuracy:', acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test score: 0.45394647121429443\n",
      "Test accuracy: 0.8408349752426147\n"
     ]
    }
   ],
   "source": [
    "print('Test score:', score)\n",
    "print('Test accuracy:', acc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Confusion Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6250/6250 - 32s\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "       asian       0.81      0.42      0.55      3876\n",
      "    hispanic       0.82      0.86      0.84     33455\n",
      "    nh_black       0.76      0.43      0.55     28290\n",
      "    nh_white       0.86      0.93      0.89    134379\n",
      "\n",
      "    accuracy                           0.84    200000\n",
      "   macro avg       0.81      0.66      0.71    200000\n",
      "weighted avg       0.83      0.84      0.83    200000\n",
      "\n",
      "[[  1612    478    236   1550]\n",
      " [    66  28643    457   4289]\n",
      " [    76    666  12290  15258]\n",
      " [   231   5291   3235 125622]]\n"
     ]
    }
   ],
   "source": [
    "p = model.predict(X_test, verbose=2) # to predict probability\n",
    "y_pred = np.argmax(p, axis=-1)\n",
    "target_names = list(sdf.race.astype('category').cat.categories)\n",
    "print(classification_report(np.argmax(y_test, axis=1), y_pred, target_names=target_names))\n",
    "print(confusion_matrix(np.argmax(y_test, axis=1), y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save('./fl_voter_reg/lstm/fl_all_name_lstm.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "words_df = pd.DataFrame(words_list, columns=['vocab'])\n",
    "words_df.to_csv('./fl_voter_reg/lstm/fl_all_name_vocab.csv', index=False, encoding='utf-8')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}