Supervised_glove_and_simple_feats_10k.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "GJu3WpO1gOlW"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pickle\n",
    "import pandas as pd\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "PkZPXM8NrnSg"
   },
   "outputs": [],
   "source": [
    "GLOVE_DIR = 'glove.6B.100d.txt'\n",
    "glove_dict = {}\n",
    "with open(GLOVE_DIR) as f:\n",
    "    for line in f:\n",
    "        word, *vector = line.split()\n",
    "        glove_dict[word]=np.array(vector).astype(float)\n",
    "\n",
    "re_tokenized = pickle.load(open('re_tokenized_lemma.pkl','rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "d7frfiHPrzdA"
   },
   "outputs": [],
   "source": [
    "def get_average_glove_vec(list_of_tokens):\n",
    "    avg_vec=[]\n",
    "    for token in list_of_tokens:\n",
    "        if token in glove_dict:\n",
    "            avg_vec.append(glove_dict[token])\n",
    "\n",
    "    if len(avg_vec)>0:\n",
    "        avg_vec = np.mean(avg_vec, axis=0)\n",
    "    else:\n",
    "        avg_vec = np.zeros(100)\n",
    "\n",
    "    return avg_vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "5vJF4m56rzfs",
    "outputId": "38395a02-75ae-4485-b093-0559afac44c1"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 416768/416768 [00:09<00:00, 41690.20it/s]\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "from pre_processing import load_df_and_features\n",
    "\n",
    "df_original = load_df_and_features(\"\")\n",
    "df_vecs = []\n",
    "\n",
    "for ls in tqdm(re_tokenized):\n",
    "    avg_vec = get_average_glove_vec(ls)\n",
    "    df_vecs.append(avg_vec)\n",
    "\n",
    "df_original['glove_avg'] = df_vecs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "id": "bi6sANfqMiDC"
   },
   "outputs": [],
   "source": [
    "num_features = ['aoa', 'concrete_score', 'verb2',\n",
    "                'conc_unknown', 'aoa_perc_known_lem', 'conc_total', 'syllable_per_word',\n",
    "                'conc_mean_score', 'dale_chall_score', 'conc_subtlex_score']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "id": "PalHqDQINXjk"
   },
   "outputs": [],
   "source": [
    "X1 = df_original[num_features].to_numpy()\n",
    "X2 = np.vstack(df_vecs)\n",
    "X = np.hstack([X1,X2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "id": "o7vveZY9Nv1b"
   },
   "outputs": [],
   "source": [
    "df = df_original.sample(10000, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "id": "rRxf6ZoOMmwW"
   },
   "outputs": [],
   "source": [
    "X_sampled = X[df.index]\n",
    "y = df['label']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "FW96NzPPOtOV",
    "outputId": "b181b58c-bad0-4d9b-9778-8d0909bbcb9e"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((10000, 110), (10000,))"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_sampled.shape, y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "id": "wTFYVcj8rzik"
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "Train_X, Test_X, Train_Y, Test_Y = train_test_split(X_sampled, y, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "id": "_TUISuFrsFrH"
   },
   "outputs": [],
   "source": [
    "# Train_X.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "kYWHduFisJfL"
   },
   "source": [
    "# Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "s2Vz5FEGsLKg",
    "outputId": "4901679b-5296-40d3-c3c6-e6fc180845aa"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.684, 0.6892822025565388)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import accuracy_score, f1_score\n",
    "\n",
    "rf = RandomForestClassifier(bootstrap=True,\n",
    " max_depth=70,\n",
    " max_features='auto',\n",
    " min_samples_leaf=4,\n",
    " min_samples_split=10,\n",
    " n_estimators=800)\n",
    "\n",
    "rf.fit(Train_X, Train_Y)\n",
    "rf_preds = rf.predict(Test_X)\n",
    "rf_acc = accuracy_score(Test_Y, rf_preds)\n",
    "rf_f1 = f1_score(Test_Y, rf_preds)\n",
    "rf_acc, rf_f1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.986"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf_training_score = rf.score(Train_X, Train_Y)\n",
    "rf_training_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "id": "J96Zh8UQsQOL"
   },
   "outputs": [],
   "source": [
    "pickle.dump(rf, open('trained_models/rf_glove_and_simplefeats_1w.pkl', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "7i_r7ArugZnd"
   },
   "outputs": [],
   "source": [
    "reloaded_rf = pickle.load(open('trained_models/rf_glove_and_simplefeats_1w.pkl', 'rb'))\n",
    "# y_score = reloaded_rf.predict_proba(Test_X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "id": "_HKBiWxzmo7v"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "0rkPMbZKsRAo"
   },
   "source": [
    "# SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "mRBys49msQQ3",
    "outputId": "0bc665dd-c9e6-4b6b-ce82-edff1dc7c821"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(62.150000000000006, 57.01306076093129)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn import svm\n",
    "from sklearn.metrics import f1_score, accuracy_score\n",
    "\n",
    "SVM = svm.SVC(C=0.5, kernel='linear', degree=3, gamma='auto')\n",
    "SVM.fit(Train_X, Train_Y)\n",
    "\n",
    "predictions_SVM = SVM.predict(Test_X)\n",
    "svm_acc = accuracy_score(Test_Y, predictions_SVM)*100\n",
    "svm_f1 = f1_score(Test_Y, predictions_SVM)*100\n",
    "svm_acc, svm_f1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6385"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "svm_training_score = SVM.score(Train_X, Train_Y)\n",
    "svm_training_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "id": "4E2bdDxH7lku"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(56.45, 57.574281539210915)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SVM2 = svm.SVC(C=0.5, kernel='rbf', degree=3, gamma='auto')\n",
    "SVM2.fit(Train_X, Train_Y)\n",
    "\n",
    "predictions_SVM2 = SVM2.predict(Test_X)\n",
    "svm_acc2 = accuracy_score(Test_Y, predictions_SVM2)*100\n",
    "svm_f12 = f1_score(Test_Y, predictions_SVM2)*100\n",
    "svm_acc2, svm_f12"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RBF kernel does not seem to improve results in this case."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "id": "lt5_rp2LsT_g"
   },
   "outputs": [],
   "source": [
    "pickle.dump(SVM, open('trained_models/svm_glove_and_simplefeats_1w.pkl', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "GMApAUocsz0g"
   },
   "outputs": [],
   "source": [
    "reloaded_svm = pickle.load(open('trained_models/svm_glove_and_simplefeats_1w.pkl', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "fY8AWU1DtNCf"
   },
   "outputs": [],
   "source": [
    "predictions_SVM = reloaded_svm.predict(Test_X[:100])\n",
    "svm_acc = accuracy_score(Test_Y, predictions_SVM)*100\n",
    "svm_f1 = f1_score(Test_Y, predictions_SVM)*100\n",
    "svm_acc, svm_f1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "4Vky7tYfzp6s"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "SAfujVyLzp9L"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "D0pjpGcetN9m"
   },
   "source": [
    "# MLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.metrics import accuracy_score, f1_score\n",
    "\n",
    "# for mlp & svm\n",
    "sc = StandardScaler()\n",
    "Train_X_scaled = sc.fit_transform(Train_X)\n",
    "Test_X_scaled = sc.transform(Test_X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.6065, 0.6196230062832285)"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.neural_network import MLPClassifier\n",
    "# with scaling\n",
    "\n",
    "\n",
    "mlp = MLPClassifier(hidden_layer_sizes=(150,), random_state=1, max_iter=300).fit(Train_X_scaled, Train_Y)\n",
    "mlp_predict = mlp.predict(Test_X_scaled)\n",
    "mlp_acc = accuracy_score(Test_Y, mlp_predict)\n",
    "mlp_f1 = f1_score(Test_Y, mlp_predict)\n",
    "mlp_acc, mlp_f1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9855"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mlp_train_score = mlp.score(Train_X_scaled, Train_Y)\n",
    "mlp_train_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "id": "F-JfgW1DtTDj"
   },
   "outputs": [],
   "source": [
    "pickle.dump(mlp, open('trained_models/mlp_glove_and_simplefeats_1w.pkl', 'wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Without feature scaling, the model perform less ideal."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Rs-GOMmgcXri",
    "outputId": "2a670b13-f7c8-474c-a623-361bf37547a5"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.5245, 0.680550890157877)"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# without scaling\n",
    "mlp2 = MLPClassifier(hidden_layer_sizes=(150,), random_state=1, max_iter=300).fit(Train_X, Train_Y)\n",
    "mlp_predict2 = mlp2.predict(Test_X)\n",
    "\n",
    "mlp_acc2 = accuracy_score(Test_Y, mlp_predict2)\n",
    "mlp_f12 = f1_score(Test_Y, mlp_predict2)\n",
    "mlp_acc2, mlp_f12"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "mlp_acc=mlp_acc*100\n",
    "mlp_f1=mlp_f1*100\n",
    "rf_acc=rf_acc*100\n",
    "rf_f1=rf_f1*100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "id": "o8KpVWG9kteP",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    accuracy\t f1 score\n",
      " mlp: 60.65 \t 61.96230062832285 \n",
      " SVM: 62.15 \t 57.01306076093129 \n",
      " rf:  68.4 \t 68.92822025565388\n"
     ]
    }
   ],
   "source": [
    "print('    accuracy\\t f1 score')\n",
    "print(' mlp:', round(mlp_acc,3),'\\t', mlp_f1, \n",
    "      '\\n SVM:', round(svm_acc,3),'\\t', svm_f1, \n",
    "      '\\n rf: ',rf_acc,'\\t', rf_f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "id": "eJ-Ff81wh99R"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>accuracy</th>\n",
       "      <th>f1 score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>MLP</th>\n",
       "      <td>60.65</td>\n",
       "      <td>61.962301</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SVM</th>\n",
       "      <td>62.15</td>\n",
       "      <td>57.013061</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RandomForest</th>\n",
       "      <td>68.40</td>\n",
       "      <td>68.928220</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              accuracy   f1 score\n",
       "MLP              60.65  61.962301\n",
       "SVM              62.15  57.013061\n",
       "RandomForest     68.40  68.928220"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score_df = pd.DataFrame([[mlp_acc, svm_acc, rf_acc],[mlp_f1, svm_f1, rf_f1]]).T\n",
    "score_df.columns=['accuracy', 'f1 score']\n",
    "score_df.index = ['MLP','SVM','RandomForest']\n",
    "score_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "id": "yPcVNTqJktbC"
   },
   "outputs": [],
   "source": [
    "score_df.to_csv('scores_1w_glove_simplefeats.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "kVMKsDo3kuWL"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "id": "w7hFjz87ktiB"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "id": "37cNuj19ktjn"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "id": "KYVtFp1Dktle"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "6dSgxvsLX81h"
   },
   "source": [
    "# Extra: Try a more complicated model under neural network, but this time using simple Count Vectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "hw18Qs6ZX_Zr"
   },
   "outputs": [],
   "source": [
    "sentences = df_original['original_text'].values\n",
    "y = df_original['label'].values\n",
    "\n",
    "sentences_train = np.array(sentences)[train_indexes]\n",
    "sentences_test = np.array(sentences)[test_indexes]\n",
    "y_train = np.array(y)[train_indexes]\n",
    "y_test = np.array(y)[test_indexes]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "IEmDQz8CY7R8"
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras.preprocessing.text import Tokenizer\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "# for Neural Network\n",
    "tokenizer = Tokenizer(num_words=10000)\n",
    "tokenizer.fit_on_texts(sentences_train)\n",
    "X_train = tokenizer.texts_to_sequences(sentences_train)\n",
    "X_test = tokenizer.texts_to_sequences(sentences_test)\n",
    "\n",
    "# Adding 1 because of  reserved 0 index\n",
    "vocab_size = len(tokenizer.word_index) + 1 \n",
    "\n",
    "maxlen = 100\n",
    "\n",
    "X_train = pad_sequences(X_train, padding='post', maxlen=maxlen)\n",
    "X_test = pad_sequences(X_test, padding='post', maxlen=maxlen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "1-h7kx-fZ6cZ",
    "outputId": "d212eb1f-782a-4783-8b6e-5c0041bf39c3"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(80000, 100)"
      ]
     },
     "execution_count": 22,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ymNmbnm3ZJKi",
    "outputId": "4f35d77e-27e5-4df4-9b43-6912757c561b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         [(None, 100)]             0         \n",
      "_________________________________________________________________\n",
      "embedding (Embedding)        (None, 100, 100)          7716400   \n",
      "_________________________________________________________________\n",
      "bidirectional (Bidirectional (None, 256)               234496    \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 32)                8224      \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 1)                 33        \n",
      "=================================================================\n",
      "Total params: 7,959,153\n",
      "Trainable params: 7,959,153\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "from keras.models import Sequential\n",
    "from keras import layers\n",
    "import tensorflow as tf\n",
    "\n",
    "embedding_dim=100\n",
    "nlp_input = layers.Input(shape=(maxlen,)) \n",
    "# meta_input = layers.Input(shape=(6,))\n",
    "emb = layers.Embedding(vocab_size, embedding_dim, input_length=maxlen)(nlp_input) \n",
    "nlp_out = layers.Bidirectional(layers.LSTM(128))(emb) \n",
    "# concat = tf.concat([nlp_out, meta_input], axis=1) \n",
    "classifier = layers.Dense(32, activation='relu')(nlp_out) \n",
    "output = layers.Dense(1, activation='sigmoid')(classifier) \n",
    "model = tf.keras.Model(inputs=[nlp_input], outputs=[output])\n",
    "\n",
    "model.compile(optimizer='adam',\n",
    "              loss='binary_crossentropy',\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "85szewVCZJND",
    "outputId": "59c826da-c199-40f9-9cbd-d2ea2ee69248"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "625/625 [==============================] - 414s 657ms/step - loss: 0.6274 - accuracy: 0.6406\n",
      "Epoch 2/10\n",
      "625/625 [==============================] - 404s 646ms/step - loss: 0.5265 - accuracy: 0.7311\n",
      "Epoch 3/10\n",
      "625/625 [==============================] - 402s 644ms/step - loss: 0.4754 - accuracy: 0.7647\n",
      "Epoch 4/10\n",
      "625/625 [==============================] - 398s 637ms/step - loss: 0.4166 - accuracy: 0.7964\n",
      "Epoch 5/10\n",
      "625/625 [==============================] - 399s 638ms/step - loss: 0.3621 - accuracy: 0.8217\n",
      "Epoch 6/10\n",
      "625/625 [==============================] - 397s 635ms/step - loss: 0.3120 - accuracy: 0.8478\n",
      "Epoch 7/10\n",
      "625/625 [==============================] - 400s 640ms/step - loss: 0.2712 - accuracy: 0.8712\n",
      "Epoch 8/10\n",
      "625/625 [==============================] - 400s 641ms/step - loss: 0.2270 - accuracy: 0.8963\n",
      "Epoch 9/10\n",
      "625/625 [==============================] - 400s 640ms/step - loss: 0.1965 - accuracy: 0.9109\n",
      "Epoch 10/10\n",
      "625/625 [==============================] - 398s 636ms/step - loss: 0.1670 - accuracy: 0.9258\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7f27c9837fd0>"
      ]
     },
     "execution_count": 23,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# model.fit({'input_1': X_train, 'input_2': meta_X_train}, y_train, epochs=10, batch_size=128)\n",
    "model.fit(X_train, y_train, epochs=10, batch_size=128)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Q_K2Yz4-ZJQE",
    "outputId": "37b0e9a7-c30f-4d63-c191-c72c16c7dac4"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:Found untraced functions such as lstm_cell_1_layer_call_and_return_conditional_losses, lstm_cell_1_layer_call_fn, lstm_cell_2_layer_call_and_return_conditional_losses, lstm_cell_2_layer_call_fn, lstm_cell_1_layer_call_fn while saving (showing 5 of 10). These functions will not be directly callable after loading.\n",
      "WARNING:absl:Found untraced functions such as lstm_cell_1_layer_call_and_return_conditional_losses, lstm_cell_1_layer_call_fn, lstm_cell_2_layer_call_and_return_conditional_losses, lstm_cell_2_layer_call_fn, lstm_cell_1_layer_call_fn while saving (showing 5 of 10). These functions will not be directly callable after loading.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: drive/MyDrive/Milestone2/trained_models/NN_tf_countvec/assets\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: drive/MyDrive/Milestone2/trained_models/NN_tf_countvec/assets\n"
     ]
    }
   ],
   "source": [
    "model.save('drive/MyDrive/Milestone2/trained_models/NN_tf_countvec')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "9kk4XtAQZJST"
   },
   "outputs": [],
   "source": [
    "predicted2 = model.predict(X_test)\n",
    "predicted = 1*(predicted2>0.5).flatten()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ChJjBVtTZd0Q"
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score, f1_score\n",
    "nn2_acc = accuracy_score(y_test, predicted)\n",
    "nn2_f1 = f1_score(y_test, predicted)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "WoCrqkTz1_-j",
    "outputId": "bad0fc53-6c53-4071-8c3e-025021e9ade6"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.64675, 0.6450997136685589)"
      ]
     },
     "execution_count": 27,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nn2_acc, nn2_f1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "RkzKL2Ce2BpP"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "Milestone2_word2vec-glove_and_simple_feats.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}