Supervised_glove_10k.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "PkZPXM8NrnSg"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pickle\n",
    "\n",
    "\n",
    "GLOVE_DIR = 'glove.6B.100d.txt'\n",
    "glove_dict = {}\n",
    "with open(GLOVE_DIR) as f:\n",
    "    for line in f:\n",
    "        word, *vector = line.split()\n",
    "        glove_dict[word]=np.array(vector).astype(float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "re_tokenized = pickle.load(open('sample_data/re_tokenized_lemma.pkl','rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "d7frfiHPrzdA"
   },
   "outputs": [],
   "source": [
    "def get_average_glove_vec(list_of_tokens):\n",
    "    avg_vec=[]\n",
    "    for token in list_of_tokens:\n",
    "        if token in glove_dict:\n",
    "            avg_vec.append(glove_dict[token])\n",
    "\n",
    "    if len(avg_vec)>0:\n",
    "        avg_vec = np.mean(avg_vec, axis=0)\n",
    "    else:\n",
    "        avg_vec = np.zeros(100)\n",
    "\n",
    "    return avg_vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "5vJF4m56rzfs",
    "outputId": "59f1c324-84e4-4c26-c252-ebb3ba475e65"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 416768/416768 [00:09<00:00, 43102.19it/s]\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "\n",
    "df_original = pd.read_csv(\"sample_data/WikiLarge_Train.csv\")\n",
    "df_vecs = []\n",
    "\n",
    "\n",
    "for ls in tqdm(re_tokenized):\n",
    "    avg_vec = get_average_glove_vec(ls)\n",
    "    df_vecs.append(avg_vec)\n",
    "\n",
    "df_original['glove_avg'] = df_vecs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "id": "wTFYVcj8rzik"
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "df = df_original.sample(10000, random_state=42)\n",
    "X = df['glove_avg']\n",
    "y = df['label']\n",
    "Train_X, Test_X, Train_Y, Test_Y = train_test_split(X,y,test_size=0.2, random_state=42)\n",
    "# save for later complex NN\n",
    "train_indexes = Train_X.index\n",
    "test_indexes = Test_X.index\n",
    "Train_X = np.vstack(Train_X)\n",
    "Test_X = np.vstack(Test_X)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "kYWHduFisJfL"
   },
   "source": [
    "# Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "s2Vz5FEGsLKg",
    "outputId": "6104f682-c5c1-4dd4-abe8-c7532213392f"
   },
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import accuracy_score, f1_score\n",
    "\n",
    "rf = RandomForestClassifier(bootstrap= True,\n",
    " max_depth=70,\n",
    " max_features='auto',\n",
    " min_samples_leaf=4,\n",
    " min_samples_split=10,\n",
    " n_estimators=800)\n",
    "\n",
    "rf.fit(Train_X, Train_Y)\n",
    "rf_preds = rf.predict(Test_X)\n",
    "rf_acc = accuracy_score(Test_Y, rf_preds)\n",
    "rf_f1 = f1_score(Test_Y, rf_preds)\n",
    "rf_acc, rf_f1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "id": "J96Zh8UQsQOL"
   },
   "outputs": [],
   "source": [
    "pickle.dump(rf, open('trained_models/rf_glove_1w.pkl', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(64.75, 66.12205670350792)"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf_acc, rf_f1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "0rkPMbZKsRAo"
   },
   "source": [
    "# SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "id": "mRBys49msQQ3"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(56.05, 59.66039467645709)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn import svm\n",
    "\n",
    "SVM = svm.SVC(C=0.5, kernel='linear', degree=3, gamma='auto')\n",
    "SVM.fit(Train_X,Train_Y)\n",
    "\n",
    "# consider as failure\n",
    "predictions_SVM = SVM.predict(Test_X)\n",
    "svm_acc = accuracy_score(Test_Y, predictions_SVM)*100\n",
    "svm_f1 = f1_score(Test_Y, predictions_SVM)*100\n",
    "svm_acc, svm_f1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_score = SVM.score(Train_X, Train_Y)*100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "59.775"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "id": "lt5_rp2LsT_g"
   },
   "outputs": [],
   "source": [
    "pickle.dump(SVM, open('trained_models/svm_glove_1w.pkl', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "GMApAUocsz0g"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60.050000000000004, 66.72219908371511)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SVM2 = svm.SVC(C=0.5, kernel='rbf', degree=3, gamma='auto')\n",
    "SVM2.fit(Train_X,Train_Y)\n",
    "\n",
    "# consider as failure\n",
    "predictions_SVM2 = SVM2.predict(Test_X)\n",
    "svm_acc2 = accuracy_score(Test_Y, predictions_SVM2)*100\n",
    "svm_f12 = f1_score(Test_Y, predictions_SVM2)*100\n",
    "svm_acc2, svm_f12"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "D0pjpGcetN9m"
   },
   "source": [
    "# MLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "uMyJqzwgtNHA",
    "outputId": "292f8b75-5a69-45ed-d328-dc885a671f01"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/annzhong/anaconda3/lib/python3.6/site-packages/sklearn/neural_network/_multilayer_perceptron.py:617: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (300) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.601, 0.5856697819314642)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.neural_network import MLPClassifier\n",
    "# from sklearn.metrics import f1_score, accuracy_score\n",
    "\n",
    "mlp = MLPClassifier(hidden_layer_sizes=(150), random_state=1, max_iter=300).fit(Train_X, Train_Y)\n",
    "mlp_predict = mlp.predict(Test_X)\n",
    "mlp_acc = accuracy_score(Test_Y, mlp_predict)\n",
    "mlp_f1 = f1_score(Test_Y, mlp_predict)\n",
    "mlp_acc, mlp_f1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "mlp_train_score = mlp.score(Train_X, Train_Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9695"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mlp_train_score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### This is obvious overfitting, try parameter tuning later"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "id": "F-JfgW1DtTDj"
   },
   "outputs": [],
   "source": [
    "pickle.dump(mlp, open('trained_models/mlp_glove_1w.pkl', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "id": "eJ-Ff81wh99R"
   },
   "outputs": [],
   "source": [
    "# mlp_acc = mlp_acc*100\n",
    "# mlp_f1 = mlp_f1*100\n",
    "# rf_acc = rf_acc*100\n",
    "# rf_f1 = rf_f1*100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "id": "RkzKL2Ce2BpP"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    accuracy\t f1 score\n",
      " mlp: 60.1 \t 58.566978193146426 \n",
      " SVM: 56.05 \t 59.66039467645709 \n",
      " rf:  64.75 \t 66.12205670350792\n"
     ]
    }
   ],
   "source": [
    "print('    accuracy\\t f1 score')\n",
    "print(' mlp:', round(mlp_acc,3),'\\t', mlp_f1, \n",
    "      '\\n SVM:', svm_acc,'\\t', svm_f1, \n",
    "      '\\n rf: ',rf_acc,'\\t', rf_f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "score_df = pd.DataFrame([[mlp_acc, svm_acc, rf_acc],[mlp_f1, svm_f1, rf_f1]]).T\n",
    "score_df.columns=['accuracy', 'f1 score']\n",
    "score_df.index = ['MLP','SVM','RandomForest']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "score_df.to_csv('scores_1w_Glove.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "name": "Milestone2_word2vec-glove.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}