Supervised_TFIDF_10k.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "6bPVlN6RcK52"
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "\n",
    "df_original = pd.read_csv('sample_data/WikiLarge_Train.csv')\n",
    "df_original['re_tokened'] = pickle.load(open('sample_data/re_tokenized_lemma.pkl', 'rb'))\n",
    "df_original['re_tokened2'] = df_original['re_tokened'].apply(lambda x: ' '.join(x))\n",
    "\n",
    "df = df_original.sample(10000, random_state=42)\n",
    "Train_X, Test_X, Train_Y, Test_Y = train_test_split(df['re_tokened2'], df['label'], \n",
    "                                                    test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "arPhrMawdfsh"
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "Tfidf_vect = TfidfVectorizer(max_features=10000, lowercase=True, stop_words='english')\n",
    "Tfidf_vect.fit(df['re_tokened2'])\n",
    "\n",
    "Train_X_Tfidf = Tfidf_vect.transform(Train_X)\n",
    "Test_X_Tfidf = Tfidf_vect.transform(Test_X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "tErWVDDxY5UR",
    "outputId": "760ade7d-3a1c-427f-e6ff-082074895fc4"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10000"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(Tfidf_vect.vocabulary_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "PO2GUAKwZBx8",
    "outputId": "c726c3f8-fa95-431c-9255-9e976f83ec1e"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((8000, 10000), (2000, 10000))"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Train_X_Tfidf.shape, Test_X_Tfidf.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "HHDuOVX-c5eq"
   },
   "source": [
    "# SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "EIZ0rxo3c45N"
   },
   "outputs": [],
   "source": [
    "from sklearn import svm\n",
    "from sklearn.metrics import accuracy_score, f1_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "LpuMWWmwdqCs",
    "outputId": "769930c7-36ef-4194-d4da-b7365f4bf5bb"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(57.8, 58.546168958742626)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SVM = svm.SVC(C=0.5, kernel='linear', degree=3, gamma='auto')\n",
    "SVM.fit(Train_X_Tfidf,Train_Y)\n",
    "\n",
    "predictions_SVM = SVM.predict(Test_X_Tfidf)\n",
    "svm_acc = accuracy_score(Test_Y, predictions_SVM)*100\n",
    "svm_f1 = f1_score(Test_Y, predictions_SVM)*100\n",
    "svm_acc, svm_f1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8265"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SVM.score(Train_X_Tfidf,Train_Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "id": "flYMODeBdvKK"
   },
   "outputs": [],
   "source": [
    "pickle.dump(SVM, open('trained_models/svm_tfidf_1w.pkl', 'wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### rbf kernel does not increase model performance."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(51.5, 67.98679867986799)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SVM2 = svm.SVC(C=0.5, kernel='rbf', degree=3, gamma='auto')\n",
    "SVM2.fit(Train_X_Tfidf,Train_Y)\n",
    "\n",
    "predictions_SVM2 = SVM2.predict(Test_X_Tfidf)\n",
    "svm_acc2 = accuracy_score(Test_Y, predictions_SVM2)*100\n",
    "svm_f12 = f1_score(Test_Y, predictions_SVM2)*100\n",
    "svm_acc2, svm_f12"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Dujx5ziYeQu0"
   },
   "source": [
    "# MLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "DFBk1wULeRvI",
    "outputId": "223ce72f-686b-4021-f92d-354babfebfd7"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MLPClassifier(hidden_layer_sizes=(150,))"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.neural_network import MLPClassifier\n",
    "\n",
    "mlp = MLPClassifier(hidden_layer_sizes=(150,))\n",
    "mlp.fit(Train_X_Tfidf, Train_Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.994625"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mlp.score(Train_X_Tfidf, Train_Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Jt1hn2D7eze9",
    "outputId": "d1327963-7a1e-4533-d5b7-75ff257a037a"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.5725, 0.585956416464891)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mlp_preds = mlp.predict(Test_X_Tfidf)\n",
    "mlp_acc = accuracy_score(Test_Y, mlp_preds)\n",
    "mlp_f1 = f1_score(Test_Y, mlp_preds)\n",
    "mlp_acc, mlp_f1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "x7u3SEycezl1"
   },
   "outputs": [],
   "source": [
    "pickle.dump(mlp, open('trained_models/mlp_tfidf_1w.pkl', 'wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "8IRNq-FNd7F_"
   },
   "source": [
    "# Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "kGcIwlDmd6cs",
    "outputId": "9f7de258-4b27-4cbf-eb9d-6a73a771484e"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.607, 0.5814696485623003)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "rf = RandomForestClassifier(bootstrap= True,\n",
    " max_depth=70,\n",
    " max_features='auto',\n",
    " min_samples_leaf=4,\n",
    " min_samples_split=10,\n",
    " n_estimators=800)\n",
    "\n",
    "rf.fit(Train_X_Tfidf, Train_Y)\n",
    "rf_preds = rf.predict(Test_X_Tfidf)\n",
    "rf_acc = accuracy_score(Test_Y, rf_preds)\n",
    "rf_f1 = f1_score(Test_Y, rf_preds)\n",
    "rf_acc, rf_f1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.74825"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf.score(Train_X_Tfidf, Train_Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "id": "8fazq4CACVrT"
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "pickle.dump(rf, open('trained_models/rf_tfidf_1w.pkl', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Summary\n",
    "\n",
    "Models with the same parameters for other features all behave like overfitting with the Tf-Idf feature. With more time one should try to do **PCA or any dimension reduction** on the input data and then feed into the model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "mlp_acc= mlp_acc*100\n",
    "mlp_f1= mlp_f1*100\n",
    "rf_acc=rf_acc*100\n",
    "rf_f1=rf_f1*100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "id": "Mju0JwQbkkGq"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    accuracy\t f1 score\n",
      " mlp: 57.25 \t 58.595641646489106 \n",
      " SVM: 57.8 \t 58.546168958742626 \n",
      " rf:  60.7 \t 58.146964856230035\n"
     ]
    }
   ],
   "source": [
    "print('    accuracy\\t f1 score')\n",
    "print(' mlp:', mlp_acc,'\\t', mlp_f1, \n",
    "      '\\n SVM:', svm_acc,'\\t', svm_f1, \n",
    "      '\\n rf: ',round(rf_acc,3),'\\t', rf_f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "score_df = pd.DataFrame([[mlp_acc, svm_acc, rf_acc],[mlp_f1, svm_f1, rf_f1]]).T\n",
    "score_df.columns=['accuracy', 'f1 score']\n",
    "score_df.index = ['MLP','SVM','RandomForest']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "score_df.to_csv('scores_1w_TFIDF.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "name": "Milestone2_TFIDF.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}