9-TFIDF_add_features_bagging_Decision_trees.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Modeling using TFIDF on `'body'`, additional features, Bagging with Decision Tree & GridSearch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.model_selection import train_test_split, GridSearchCV, cross_val_score\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.feature_extraction import stop_words\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from tokenizer.tokenizer import RedditTokenizer\n",
    "from sklearn.ensemble import BaggingClassifier\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "import nltk\n",
    "from nltk import word_tokenize  \n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import WordNetLemmatizer \n",
    "from nltk.stem.porter import PorterStemmer\n",
    "\n",
    "import regex\n",
    "import time\n",
    "\n",
    "from my_tools.pipes_grids import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "nltk_stops = set(stopwords.words('english'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    df = pd.read_csv('datasets/data_submissions.csv')\n",
    "except:\n",
    "    df = pd.read_csv('https://www.dropbox.com/s/eh48hdw4af7tjc3/data_submissions.csv?dl=0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('datasets/df_sample.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define the pipeline and gridsearch here to be used for this round of analysis."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "ct = ColumnTransformer(\n",
    "    [('tvec',TfidfVectorizer(), 'body')],\n",
    "    remainder='passthrough',\n",
    "    sparse_threshold=0.3,\n",
    "    n_jobs=-1,\n",
    ")\n",
    "\n",
    "pipe = Pipeline([\n",
    "    ('ct',ct),\n",
    "    ('bagclass',  BaggingClassifier(n_jobs=-1))\n",
    "])\n",
    "\n",
    "pipe_params = {\n",
    "    'ct__tvec__tokenizer' : [RedditIt()],\n",
    "    'ct__tvec__ngram_range' : [(1,1),(1,2)],\n",
    "    'ct__tvec__max_features' : [None],\n",
    "    'ct__tvec__min_df' : [.02,.05],\n",
    "    'ct__tvec__max_df' : [.9,.85],\n",
    "    'ct__tvec__stop_words' : [nltk_stops],\n",
    "    'bagclass__n_estimators' : [10],\n",
    "\n",
    "    \n",
    "\n",
    "    \n",
    "    \n",
    "    \n",
    "}\n",
    "\n",
    "\n",
    "gs = GridSearchCV(pipe, # what object are we optimizing?\n",
    "                  pipe_params, # what parameters values are we searching?\n",
    "                  cv=3,# 3-fold cross-validation.\n",
    "                 n_jobs = -1) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define functions to automate the model fitting, gridsearch & score outputs\n",
    "see [`my_tools/pipes_grids.py`](my_tools/pipes_grids.py)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Creation and Scoring"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### All Comments and Submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best score: 0.7469798356312031\n",
      "Training score: 0.9892666277450937\n",
      "Test score: 0.7518234865061999\n",
      "Cross val score: 0.7498318515259561\n",
      "Baseline Accuracy: 0.5\n",
      "Accuracy: 0.7518234865061999\n",
      "Misclassification rate: 0.2481765134938001\n",
      "Sensitivity & Recall: 0.6734135667396062\n",
      "Specificity: 0.8302334062727936\n",
      "Precision: 0.798659169550173\n",
      "Balanced Accuracy Score: 0.5036469730123998\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'bagclass__n_estimators': 10,\n",
       " 'ct__tvec__max_df': 0.9,\n",
       " 'ct__tvec__max_features': None,\n",
       " 'ct__tvec__min_df': 0.02,\n",
       " 'ct__tvec__ngram_range': (1, 2),\n",
       " 'ct__tvec__stop_words': {'a',\n",
       "  'about',\n",
       "  'above',\n",
       "  'after',\n",
       "  'again',\n",
       "  'against',\n",
       "  'ain',\n",
       "  'all',\n",
       "  'am',\n",
       "  'an',\n",
       "  'and',\n",
       "  'any',\n",
       "  'are',\n",
       "  'aren',\n",
       "  \"aren't\",\n",
       "  'as',\n",
       "  'at',\n",
       "  'be',\n",
       "  'because',\n",
       "  'been',\n",
       "  'before',\n",
       "  'being',\n",
       "  'below',\n",
       "  'between',\n",
       "  'both',\n",
       "  'but',\n",
       "  'by',\n",
       "  'can',\n",
       "  'couldn',\n",
       "  \"couldn't\",\n",
       "  'd',\n",
       "  'did',\n",
       "  'didn',\n",
       "  \"didn't\",\n",
       "  'do',\n",
       "  'does',\n",
       "  'doesn',\n",
       "  \"doesn't\",\n",
       "  'doing',\n",
       "  'don',\n",
       "  \"don't\",\n",
       "  'down',\n",
       "  'during',\n",
       "  'each',\n",
       "  'few',\n",
       "  'for',\n",
       "  'from',\n",
       "  'further',\n",
       "  'had',\n",
       "  'hadn',\n",
       "  \"hadn't\",\n",
       "  'has',\n",
       "  'hasn',\n",
       "  \"hasn't\",\n",
       "  'have',\n",
       "  'haven',\n",
       "  \"haven't\",\n",
       "  'having',\n",
       "  'he',\n",
       "  'her',\n",
       "  'here',\n",
       "  'hers',\n",
       "  'herself',\n",
       "  'him',\n",
       "  'himself',\n",
       "  'his',\n",
       "  'how',\n",
       "  'i',\n",
       "  'if',\n",
       "  'in',\n",
       "  'into',\n",
       "  'is',\n",
       "  'isn',\n",
       "  \"isn't\",\n",
       "  'it',\n",
       "  \"it's\",\n",
       "  'its',\n",
       "  'itself',\n",
       "  'just',\n",
       "  'll',\n",
       "  'm',\n",
       "  'ma',\n",
       "  'me',\n",
       "  'mightn',\n",
       "  \"mightn't\",\n",
       "  'more',\n",
       "  'most',\n",
       "  'mustn',\n",
       "  \"mustn't\",\n",
       "  'my',\n",
       "  'myself',\n",
       "  'needn',\n",
       "  \"needn't\",\n",
       "  'no',\n",
       "  'nor',\n",
       "  'not',\n",
       "  'now',\n",
       "  'o',\n",
       "  'of',\n",
       "  'off',\n",
       "  'on',\n",
       "  'once',\n",
       "  'only',\n",
       "  'or',\n",
       "  'other',\n",
       "  'our',\n",
       "  'ours',\n",
       "  'ourselves',\n",
       "  'out',\n",
       "  'over',\n",
       "  'own',\n",
       "  're',\n",
       "  's',\n",
       "  'same',\n",
       "  'shan',\n",
       "  \"shan't\",\n",
       "  'she',\n",
       "  \"she's\",\n",
       "  'should',\n",
       "  \"should've\",\n",
       "  'shouldn',\n",
       "  \"shouldn't\",\n",
       "  'so',\n",
       "  'some',\n",
       "  'such',\n",
       "  't',\n",
       "  'than',\n",
       "  'that',\n",
       "  \"that'll\",\n",
       "  'the',\n",
       "  'their',\n",
       "  'theirs',\n",
       "  'them',\n",
       "  'themselves',\n",
       "  'then',\n",
       "  'there',\n",
       "  'these',\n",
       "  'they',\n",
       "  'this',\n",
       "  'those',\n",
       "  'through',\n",
       "  'to',\n",
       "  'too',\n",
       "  'under',\n",
       "  'until',\n",
       "  'up',\n",
       "  've',\n",
       "  'very',\n",
       "  'was',\n",
       "  'wasn',\n",
       "  \"wasn't\",\n",
       "  'we',\n",
       "  'were',\n",
       "  'weren',\n",
       "  \"weren't\",\n",
       "  'what',\n",
       "  'when',\n",
       "  'where',\n",
       "  'which',\n",
       "  'while',\n",
       "  'who',\n",
       "  'whom',\n",
       "  'why',\n",
       "  'will',\n",
       "  'with',\n",
       "  'won',\n",
       "  \"won't\",\n",
       "  'wouldn',\n",
       "  \"wouldn't\",\n",
       "  'y',\n",
       "  'you',\n",
       "  \"you'd\",\n",
       "  \"you'll\",\n",
       "  \"you're\",\n",
       "  \"you've\",\n",
       "  'your',\n",
       "  'yours',\n",
       "  'yourself',\n",
       "  'yourselves'},\n",
       " 'ct__tvec__tokenizer': <my_tools.pipes_grids.RedditIt at 0x10419af90>}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = df[['body']]\n",
    "y = df['is_2016']\n",
    "model_base = model_go(X=X,\n",
    "        y=y,\n",
    "      gridsearch=gs)\n",
    "model_base[0].best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best score: 0.7496070418107513\n",
      "Training score: 0.9886828041496385\n",
      "Test score: 0.7529631655725748\n",
      "Cross val score: 0.7473167890024546\n",
      "Baseline Accuracy: 0.5\n",
      "Accuracy: 0.7529631655725748\n",
      "Misclassification rate: 0.2470368344274252\n",
      "Sensitivity & Recall: 0.6741429613420861\n",
      "Specificity: 0.8317833698030634\n",
      "Precision: 0.8003030631020673\n",
      "Balanced Accuracy Score: 0.5059263311451496\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'bagclass__n_estimators': 10,\n",
       " 'ct__tvec__max_df': 0.9,\n",
       " 'ct__tvec__max_features': None,\n",
       " 'ct__tvec__min_df': 0.02,\n",
       " 'ct__tvec__ngram_range': (1, 2),\n",
       " 'ct__tvec__stop_words': {'a',\n",
       "  'about',\n",
       "  'above',\n",
       "  'after',\n",
       "  'again',\n",
       "  'against',\n",
       "  'ain',\n",
       "  'all',\n",
       "  'am',\n",
       "  'an',\n",
       "  'and',\n",
       "  'any',\n",
       "  'are',\n",
       "  'aren',\n",
       "  \"aren't\",\n",
       "  'as',\n",
       "  'at',\n",
       "  'be',\n",
       "  'because',\n",
       "  'been',\n",
       "  'before',\n",
       "  'being',\n",
       "  'below',\n",
       "  'between',\n",
       "  'both',\n",
       "  'but',\n",
       "  'by',\n",
       "  'can',\n",
       "  'couldn',\n",
       "  \"couldn't\",\n",
       "  'd',\n",
       "  'did',\n",
       "  'didn',\n",
       "  \"didn't\",\n",
       "  'do',\n",
       "  'does',\n",
       "  'doesn',\n",
       "  \"doesn't\",\n",
       "  'doing',\n",
       "  'don',\n",
       "  \"don't\",\n",
       "  'down',\n",
       "  'during',\n",
       "  'each',\n",
       "  'few',\n",
       "  'for',\n",
       "  'from',\n",
       "  'further',\n",
       "  'had',\n",
       "  'hadn',\n",
       "  \"hadn't\",\n",
       "  'has',\n",
       "  'hasn',\n",
       "  \"hasn't\",\n",
       "  'have',\n",
       "  'haven',\n",
       "  \"haven't\",\n",
       "  'having',\n",
       "  'he',\n",
       "  'her',\n",
       "  'here',\n",
       "  'hers',\n",
       "  'herself',\n",
       "  'him',\n",
       "  'himself',\n",
       "  'his',\n",
       "  'how',\n",
       "  'i',\n",
       "  'if',\n",
       "  'in',\n",
       "  'into',\n",
       "  'is',\n",
       "  'isn',\n",
       "  \"isn't\",\n",
       "  'it',\n",
       "  \"it's\",\n",
       "  'its',\n",
       "  'itself',\n",
       "  'just',\n",
       "  'll',\n",
       "  'm',\n",
       "  'ma',\n",
       "  'me',\n",
       "  'mightn',\n",
       "  \"mightn't\",\n",
       "  'more',\n",
       "  'most',\n",
       "  'mustn',\n",
       "  \"mustn't\",\n",
       "  'my',\n",
       "  'myself',\n",
       "  'needn',\n",
       "  \"needn't\",\n",
       "  'no',\n",
       "  'nor',\n",
       "  'not',\n",
       "  'now',\n",
       "  'o',\n",
       "  'of',\n",
       "  'off',\n",
       "  'on',\n",
       "  'once',\n",
       "  'only',\n",
       "  'or',\n",
       "  'other',\n",
       "  'our',\n",
       "  'ours',\n",
       "  'ourselves',\n",
       "  'out',\n",
       "  'over',\n",
       "  'own',\n",
       "  're',\n",
       "  's',\n",
       "  'same',\n",
       "  'shan',\n",
       "  \"shan't\",\n",
       "  'she',\n",
       "  \"she's\",\n",
       "  'should',\n",
       "  \"should've\",\n",
       "  'shouldn',\n",
       "  \"shouldn't\",\n",
       "  'so',\n",
       "  'some',\n",
       "  'such',\n",
       "  't',\n",
       "  'than',\n",
       "  'that',\n",
       "  \"that'll\",\n",
       "  'the',\n",
       "  'their',\n",
       "  'theirs',\n",
       "  'them',\n",
       "  'themselves',\n",
       "  'then',\n",
       "  'there',\n",
       "  'these',\n",
       "  'they',\n",
       "  'this',\n",
       "  'those',\n",
       "  'through',\n",
       "  'to',\n",
       "  'too',\n",
       "  'under',\n",
       "  'until',\n",
       "  'up',\n",
       "  've',\n",
       "  'very',\n",
       "  'was',\n",
       "  'wasn',\n",
       "  \"wasn't\",\n",
       "  'we',\n",
       "  'were',\n",
       "  'weren',\n",
       "  \"weren't\",\n",
       "  'what',\n",
       "  'when',\n",
       "  'where',\n",
       "  'which',\n",
       "  'while',\n",
       "  'who',\n",
       "  'whom',\n",
       "  'why',\n",
       "  'will',\n",
       "  'with',\n",
       "  'won',\n",
       "  \"won't\",\n",
       "  'wouldn',\n",
       "  \"wouldn't\",\n",
       "  'y',\n",
       "  'you',\n",
       "  \"you'd\",\n",
       "  \"you'll\",\n",
       "  \"you're\",\n",
       "  \"you've\",\n",
       "  'your',\n",
       "  'yours',\n",
       "  'yourself',\n",
       "  'yourselves'},\n",
       " 'ct__tvec__tokenizer': <my_tools.pipes_grids.RedditIt at 0x10419af90>}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = df[['body','vaderSentiment']]\n",
    "y = df['is_2016']\n",
    "model_vader = model_go(X=X,\n",
    "        y=y,\n",
    "        gridsearch=gs)\n",
    "model_vader[0].best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "X = df[['body','senti_score']]\n",
    "y = df['is_2016']\n",
    "model_sentiscore = model_go(X=X,\n",
    "        y=y,\n",
    "        gridsearch=gs)\n",
    "model_sentiscore[0].best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:.conda-dsi] *",
   "language": "python",
   "name": "conda-env-.conda-dsi-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}