Algoritmos_w_Pipelines.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "%matplotlib inline\n",
    "import datetime\n",
    "import re\n",
    "import string\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "import xgboost as xgb\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn.model_selection import GridSearchCV,StratifiedKFold,RandomizedSearchCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.metrics import f1_score\n",
    "from wordcloud import WordCloud,STOPWORDS\n",
    "from sklearn.model_selection import train_test_split\n",
    "from nltk.stem.snowball import SnowballStemmer\n",
    "import nltk, os, re, string, collections\n",
    "import matplotlib\n",
    "from  imageio import imread\n",
    "from nltk.util import ngrams\n",
    "import itertools\n",
    "import spacy\n",
    "from sklearn.preprocessing import MultiLabelBinarizer, Binarizer\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "from sklearn import metrics\n",
    "from sklearn.metrics import classification_report, confusion_matrix, accuracy_score\n",
    "from collections import Counter\n",
    "from sklearn.model_selection import train_test_split,KFold\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.svm import SVR\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.metrics import classification_report, confusion_matrix, accuracy_score\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.multioutput import MultiOutputClassifier\n",
    "from sklearn.model_selection import RepeatedStratifiedKFold\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn import metrics\n",
    "import lightgbm as lgb\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "from sklearn.model_selection import cross_val_predict, cross_val_score\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\", category=FutureWarning)\n",
    "warnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n",
    "import gc\n",
    "from mlxtend.classifier import StackingClassifier\n",
    "from bayes_opt import BayesianOptimization\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.pipeline import FeatureUnion\n",
    "import pickle\n",
    "from sklearn.exceptions import ConvergenceWarning\n",
    "warnings.filterwarnings(\"ignore\", category=ConvergenceWarning)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv('original_data/train.csv')\n",
    "test  = pd.read_csv('original_data/test.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1) LIMPIEZA DE TEXTO"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1) a) Funciones"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2253\n"
     ]
    }
   ],
   "source": [
    "def getStopwordsList():\n",
    "    fileNamesList = [\"texts/99webtools.txt\", \"texts/atire_ncbi.txt\", \"texts/atire_puurula.txt\", \"texts/azure.txt\", \n",
    "                 \"texts/bbalet.txt\", \n",
    "                 \"texts/bow_short.txt\", \"texts/choi_2000naacl.txt\", \"texts/cook1988_function_words.txt\", \n",
    "                 \"texts/corenlp_acronym.txt\", \n",
    "                 \"texts/corenlp_hardcoded.txt\", \"texts/corenlp_stopwords.txt\", \"texts/datasciencedojo.txt\", \n",
    "                 \"texts/deeplearning4j.txt\", \n",
    "                 \"texts/dkpro.txt\", \"texts/mongodb.txt\", \"texts/galago_inquery.txt\", \"texts/gate_keyphrase.txt\", \n",
    "                 \"texts/gensim.txt\", \n",
    "                 \"texts/glasgow_stop_words.txt\", \"texts/indri.txt\", \"texts/kevinbouge.txt\", \"texts/lexisnexis.txt\",\n",
    "                 \"texts/lingpipe.txt\", \n",
    "                 \"texts/mallet.txt\", \"texts/mysql_innodb.txt\", \"texts/mysql_myisam.txt\", \"texts/galago_rmstop.txt\", \n",
    "                 \"texts/atire_ncbi.txt\", \n",
    "                 \"texts/galago_rmstop.txt\", \"texts/nltk.txt\", \"texts/okapiframework.txt\", \"texts/okapi_cacm_expanded.txt\", \n",
    "                 \"texts/onix.txt\", \n",
    "                 \"texts/ovid.txt\", \"texts/postgresql.txt\", \"texts/pubmed.txt\", \"texts/quanteda.txt\", \"texts/r_tm.txt\", \n",
    "                 \"texts/ranksnl_large.txt\", \n",
    "                 \"texts/reuters_wos.txt\", \"texts/rouge_155.txt\", \"texts/scikitlearn.txt\", \"texts/smart.txt\", \n",
    "                 \"texts/snowball_expanded.txt\", \n",
    "                 \"texts/spacy.txt\", \"texts/spark_mllib.txt\", \"texts/sphinx_mirasvit.txt\", \"texts/t101_minimal.txt\", \n",
    "                 \"texts/taporware.txt\", \n",
    "                 \"texts/terrier.txt\", \"texts/tonybsk_1.txt\", \"texts/tonybsk_6.txt\", \"texts/voyant_taporware.txt\", \n",
    "                 \"texts/weka.txt\", \n",
    "                 \"texts/xapian.txt\", \"texts/xpo6.txt\", \"texts/zettair.txt\"]\n",
    "    stopwordsList = []\n",
    "    for fileName in fileNamesList:\n",
    "        file = open(fileName, \"r\", encoding=\"utf8\")\n",
    "        for line in file:\n",
    "            stripped_line = line. strip()\n",
    "            line_list = stripped_line\n",
    "            if line_list not in stopwordsList:\n",
    "                stopwordsList.append(line_list)\n",
    "        file.close()\n",
    "    return stopwordsList\n",
    "\n",
    "stopwordsList = getStopwordsList()\n",
    "\n",
    "print(len(stopwordsList))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_text_strict(text):\n",
    "    text = text.lower()\n",
    "    text = text.replace('\\d+', '')\n",
    "    invalid_chars = ['#','|','@','!','?','-','_','[',']','%','&',':','.',',',\"''\",'/','https','(','//t',')','http',\n",
    "                 ';','\\'']\n",
    "    for char in invalid_chars:\n",
    "        if char in text:\n",
    "            text = text.replace(char,' ')\n",
    "    #removes url and tags\n",
    "    text = re.sub(r\"(?:\\@|https?\\://)\\S+\", \" \",  text)\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "# FUNCIONES AUXILIARES\n",
    "\n",
    "w_tokenizer = nltk.tokenize.WhitespaceTokenizer()\n",
    "lemmatizer = nltk.stem.WordNetLemmatizer()\n",
    "\n",
    "def prettify(word):\n",
    "    word = re.sub(r'[^A-Za-z]', \" \",  word)\n",
    "    word = re.sub(r'\\b\\w\\b',' ', word)\n",
    "    word = re.sub(r'\\s+', ' ', word)\n",
    "    return word\n",
    "\n",
    "def cleanStopwords(tweet):\n",
    "    tweet = tweet.split(' ')\n",
    "    cleanTweet=\"\"\n",
    "    for word in tweet:\n",
    "        if word not in stopwordsList:\n",
    "            cleanTweet = cleanTweet + \" \" + word\n",
    "    return cleanTweet\n",
    "\n",
    "def clean_text_non_strict(text): \n",
    "    tw = \" \"\n",
    "    words = text.split(' ')\n",
    "    for word in words:\n",
    "        word = ''.join(filter(lambda x: x in set(string.printable), word))\n",
    "        word = word.replace(\"\\n\",\" \")\n",
    "        word = word.replace('û',\"\")\n",
    "        word = word.replace('Û',\"\")\n",
    "        #word = word.replace('_','')\n",
    "        #word = word.replace(\"\\\"\",'')\n",
    "        #word = word.strip('.')\n",
    "        #word = word.strip(',')\n",
    "        #word = word.strip(':')\n",
    "        tw += word + \" \"\n",
    "    return tw"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp = spacy.load('en', disable=['parser', 'ner'])\n",
    "def lemmatize_text_v2(text):\n",
    "    if text == '' or pd.isnull(text):\n",
    "        return text\n",
    "    else:\n",
    "        doc = nlp(text)\n",
    "        return \" \".join([token.lemma_ for token in doc])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#FUNCIONES AUXILIARES: Limpieza de tweet\n",
    "\n",
    "# Función que saca los links\n",
    "\n",
    "def quitar_link_twitter(tweet):\n",
    "    res = []\n",
    "    for w in tweet.split(\" \"):\n",
    "        if ((\"http\" not in w) and (\"https\" not in w)):\n",
    "            res.append(w)\n",
    "    return (\" \").join(res)\n",
    "\n",
    "# Función que saca las menciones\n",
    "\n",
    "def quitar_mencion_twitter(tweet):\n",
    "    res = []\n",
    "    for w in tweet.split(\" \"):\n",
    "        if(\"@\" not in w):\n",
    "            res.append(w)\n",
    "    return (\" \").join(res)\n",
    "\n",
    "def quitar_simbolo(tweet, simbolo):\n",
    "    text = \"\"\n",
    "    for c in tweet:\n",
    "        if(c!=simbolo):\n",
    "            text+=c\n",
    "    return text\n",
    "\n",
    "def agregar_espacio_ente_comas(tweet):\n",
    "    \n",
    "    tweer_sin_coma = (\" \").join(tweet.split(\",\"))\n",
    "\n",
    "    return (\" \").join(tweer_sin_coma.split(\".\"))\n",
    "\n",
    "def capitalize_each_word(string):\n",
    "    \n",
    "    text = \"\"\n",
    "    for w in string.split(\" \"):\n",
    "        if(len(w)>3):\n",
    "            text += w.capitalize()+\" \"\n",
    "    return text\n",
    "\n",
    "def limpiar_tweet_location(tweet):\n",
    "    res = tweet\n",
    "    func = [quitar_mencion_twitter, quitar_link_twitter, agregar_espacio_ente_comas, lambda x: quitar_simbolo(x, \"#\"), capitalize_each_word]\n",
    "    for f in func:\n",
    "        res = f(res)\n",
    "    return res\n",
    "\n",
    "def to_string(a_list, delimeter):\n",
    "    res = \"\"\n",
    "    \n",
    "    for e in sorted(filter(None, a_list)):\n",
    "        res += str(e)+delimeter\n",
    "    \n",
    "    if(len(res)==0): return None\n",
    "    return res[: -len(delimeter)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def addKeywordToText(x):\n",
    "    y=pd.isnull(x.keyword)\n",
    "    if (y == True):\n",
    "        return x['text']\n",
    "    else:\n",
    "        return (x['keyword'] + ' ' + x['text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def addLocationToText(x):\n",
    "    if (x.location == ' '):\n",
    "        return x['text_w_location']\n",
    "    else:\n",
    "        return (x['location'] + ' ' + x['text_w_location'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1) b) Text + Keyword [baja un toque el score]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''train[\"keyword\"] = train[\"keyword\"].str.replace('%20',' ')\n",
    "train['text_w_keyword'] = train.apply(lambda x: addKeywordToText(x), axis=1)\n",
    "\n",
    "test[\"keyword\"] = test[\"keyword\"].str.replace('%20',' ')\n",
    "test['text_w_keyword'] = test.apply(lambda x: addKeywordToText(x), axis=1)\n",
    "\n",
    "train.text_w_keyword = train[\"text_w_keyword\"].apply(lambda x: clean_text_strict(x))\n",
    "test.text_w_keyword  = test[\"text_w_keyword\"].apply(lambda x: clean_text_strict(x))\n",
    "\n",
    "train.text_w_keyword = train.text_w_keyword.apply(prettify)\n",
    "test.text_w_keyword  = test.text_w_keyword.apply(prettify)\n",
    "\n",
    "train.text_w_keyword = train.text_w_keyword.apply(lemmatize_text_v2)\n",
    "test.text_w_keyword  = test.text_w_keyword.apply(lemmatize_text_v2)\n",
    "\n",
    "train.text_w_keyword = train.text_w_keyword.apply(cleanStopwords)\n",
    "test.text_w_keyword  = test.text_w_keyword.apply(cleanStopwords)'''"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1) c) Text + Location [baja score :(]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''train.location = train.location.fillna('')\n",
    "train.location = train.location.apply(lambda x:limpiar_tweet_location(x))\n",
    "train['text_w_location'] = train.apply(lambda x: addLocationToText(x), axis=1)\n",
    "\n",
    "test.location = test.location.fillna('')\n",
    "test.location = test.location.apply(lambda x:limpiar_tweet_location(x))\n",
    "test['text_w_location'] = test.apply(lambda x: addLocationToText(x), axis=1)\n",
    "\n",
    "train.text_w_location = train[\"text_w_location\"].apply(lambda x: clean_text_strict(x))\n",
    "test.text_w_location  = test[\"text_w_location\"].apply(lambda x: clean_text_strict(x))\n",
    "\n",
    "train.text_w_location = train.text_w_location.apply(prettify)\n",
    "test.text_w_location  = test.text_w_location.apply(prettify)\n",
    "\n",
    "train.text_w_location = train.text_w_location.apply(lemmatize_text_v2)\n",
    "test.text_w_location  = test.text_w_location.apply(lemmatize_text_v2)\n",
    "\n",
    "train.text_w_location = train.text_w_location.apply(cleanStopwords)\n",
    "test.text_w_location  = test.text_w_location.apply(cleanStopwords)'''"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1) d)  Sólo Texto [mejor hasta ahora]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#train.text = train[\"text\"].apply(lambda x: quitar_link_twitter(x)) #Por algun motivo esto baja un poquito el score\n",
    "#test.text  = test[\"text\"].apply(lambda x: quitar_link_twitter(x))\n",
    "\n",
    "#train.text = train[\"text\"].apply(lambda x: quitar_mencion_twitter(x)) #Por algun motivo esto baja un poquito el score\n",
    "#test.text  = test[\"text\"].apply(lambda x: quitar_mencion_twitter(x))\n",
    "\n",
    "train.text = train[\"text\"].apply(lambda x: clean_text_non_strict(x))\n",
    "test.text  = test[\"text\"].apply(lambda x: clean_text_non_strict(x))\n",
    "\n",
    "train.text = train[\"text\"].apply(lambda x: clean_text_strict(x))\n",
    "test.text  = test[\"text\"].apply(lambda x: clean_text_strict(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train.text = train.text.apply(prettify)\n",
    "test.text  = test.text.apply(prettify)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# a veces baja score\n",
    "#train.text = train.text.apply(lemmatize_text_v2)\n",
    "#test.text  = test.text.apply(lemmatize_text_v2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0        -PRON- deed reason earthquake allah forgive -...\n",
       "1                           forest fire ronge sask canada\n",
       "2        resident shelter notify officer evacuation sh...\n",
       "3           people receive wildfire evacuation california\n",
       "4                   photo ruby alaska smoke wildfire pour\n",
       "                              ...                        \n",
       "7608     giant crane hold bridge collapse nearby stfmb...\n",
       "7609       aria ahrary thetawniest control wild fire c...\n",
       "7610                        utc volcano hawaii zdtoyd ebj\n",
       "7611     police bike collide portugal bike rider suffe...\n",
       "7612       raze northern california wildfire abc ymy rskq\n",
       "Name: text, Length: 7613, dtype: object"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "train.text = train.text.apply(cleanStopwords)\n",
    "test.text  = test.text.apply(cleanStopwords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0       our deeds are the reason of this earthquake ma...\n",
       "1                   forest fire near la ronge sask canada\n",
       "2       all residents asked to shelter in place are be...\n",
       "3        people receive wildfires evacuation orders in...\n",
       "4       just got sent this photo from ruby alaska as s...\n",
       "                              ...                        \n",
       "7608    two giant cranes holding bridge collapse into ...\n",
       "7609     aria ahrary thetawniest the out of control wi...\n",
       "7610               utc km of volcano hawaii co zdtoyd ebj\n",
       "7611    police investigating after an bike collided wi...\n",
       "7612    the latest more homes razed by northern califo...\n",
       "Name: text, Length: 7613, dtype: object"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2) FEATURES NUMERICOS"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2) a) Preparation of the TRAIN and TEST set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TEST SET - ONLY NUMERICAL\n",
    "test_keywords_numerical = pd.read_csv('csv_lio/test/keywords_numerical_features_test - Copy.csv')\n",
    "test_keywords_numerical = test_keywords_numerical.drop(columns = 'Unnamed: 0')\n",
    "test_text_numerical = pd.read_csv('csv_lio/test/text_general_numerical_features_test - Copy.csv')\n",
    "test_text_numerical = test_text_numerical.drop(columns = ['Unnamed: 0','#silabas'])\n",
    "test_hashtag_numerical = pd.read_csv('csv_lio/test/features_hashtags_numerical.csv')\n",
    "test_hashtag_numerical  =test_hashtag_numerical.drop(columns = 'Unnamed: 0')\n",
    "test_links_numerical = pd.read_csv('csv_lio/test/features_links_numerical.csv')\n",
    "test_links_numerical = test_links_numerical.drop(columns = ['id.1','target'])\n",
    "test_new_use = test.drop(columns = ['keyword','location'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "features_test = pd.merge(test_new_use,test_text_numerical, on = 'id', how = 'left')\n",
    "features_test = pd.merge(features_test,test_keywords_numerical, on = 'id', how = 'left')\n",
    "features_test = pd.merge(features_test,test_hashtag_numerical, on = 'id', how = 'left')\n",
    "features_test = pd.merge(features_test,test_links_numerical, on = 'id', how = 'left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TEST: Df with general numerical features\n",
    "features_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TRAIN SET - ONLY NUMERICAL\n",
    "train_keywords_numerical = pd.read_csv('csv_lio/train/keywords_numerical_features - Copy.csv')\n",
    "train_keywords_numerical = train_keywords_numerical.drop(columns = ['Unnamed: 0','target'], axis = 1)\n",
    "train_text_numerical = pd.read_csv('csv_lio/train/text_general_numerical_features_train - Copy.csv')\n",
    "train_text_numerical = train_text_numerical.drop(columns = ['Unnamed: 0','target','#silabas'])\n",
    "train_hashtag_numerical = pd.read_csv('csv_lio/train/features_hashtags_numerical.csv')\n",
    "train_hashtag_numerical  =train_hashtag_numerical.drop(columns = ['Unnamed: 0','target'])\n",
    "train_links_numerical = pd.read_csv('csv_lio/train/features_links_numerical.csv')\n",
    "train_links_numerical = train_links_numerical.drop(columns = ['id.1','target'])\n",
    "train_new_use = train.drop(columns = ['keyword','location'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "features_train = pd.merge(train_new_use,train_text_numerical, on = 'id', how = 'left')\n",
    "features_train = pd.merge(features_train,train_keywords_numerical, on = 'id', how = 'left')\n",
    "features_train = pd.merge(features_train,train_hashtag_numerical, on = 'id', how = 'left')\n",
    "features_train = pd.merge(features_train,train_links_numerical, on = 'id', how = 'left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TRAIN: Df with general numerical features\n",
    "features_train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2) b) Transformer-creation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Text transformer: it selects a single column from the data frame to perform additional transformations on\n",
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "class TextSelector(BaseEstimator, TransformerMixin):\n",
    "\n",
    "    def __init__(self, key):\n",
    "        self.key = key\n",
    "\n",
    "    def fit(self, X, y=None):\n",
    "        return self\n",
    "\n",
    "    def transform(self, X):\n",
    "        return X[self.key]\n",
    "\n",
    "# Value transformer:  it selects a single column from the data frame to perform additional transformations on\n",
    "class NumberSelector(BaseEstimator, TransformerMixin):\n",
    "\n",
    "    def __init__(self, key):\n",
    "        self.key = key\n",
    "\n",
    "    def fit(self, X, y=None):\n",
    "        return self\n",
    "\n",
    "    def transform(self, X):\n",
    "        return X[[self.key]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2) d) Now, a PIPELINE for each feature in the df is created"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For TEXT -this pipeline consists of \"selecting\" and then \"bow-ing\" a column.\n",
    "bow_pipeline = Pipeline([\n",
    "                ('selector', TextSelector(key='text')),\n",
    "                ('bow',CountVectorizer(analyzer='word',binary=True,ngram_range=(1,2)))\n",
    "            ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For TEXT -  this pipeline consists of \"selecting\" and then \"td_idf-ing\" a column.\n",
    "tfidf_pipeline = Pipeline([\n",
    "                ('selector', TextSelector(key='text')),\n",
    "                ('tfidf', TfidfVectorizer( ))\n",
    "            ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For TEXT -  this pipeline consists of \"selecting\" and then \"td_idf-ing\" a column.\n",
    "tfidf2_pipeline = Pipeline([\n",
    "                ('selector', TextSelector(key='text')),\n",
    "                ('tfidf', TfidfVectorizer(use_idf=True,smooth_idf=True,ngram_range=(1,1),stop_words='english'))\n",
    "            ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For TEXT -  this pipeline consists of \"selecting\" and then \"td_idf-ing\" a column.\n",
    "tfidf3_pipeline = Pipeline([\n",
    "                ('selector', TextSelector(key='text')),\n",
    "                ('tfidf', TfidfVectorizer(use_idf=True,smooth_idf=True,ngram_range=(1,2),stop_words='english'))\n",
    "            ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For TEXT -  this pipeline consists of \"selecting\" and then \"td_idf-ing\" a column.\n",
    "tfidf4_pipeline = Pipeline([\n",
    "                ('selector', TextSelector(key='text')),\n",
    "                ('tfidf', TfidfVectorizer(use_idf=True,smooth_idf=True,ngram_range=(1,2)))\n",
    "            ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def numerical_pipeline_creator(df_train):\n",
    "    #df.columns.tolist()\n",
    "    feature_names = df_train.columns.tolist()\n",
    "    pipelines = []\n",
    "    for feature_name in feature_names:\n",
    "        current_pipeline =  Pipeline([\n",
    "                            ('selector', NumberSelector(key=feature_name)),\n",
    "                            #('standard', StandardScaler())\n",
    "                        ])\n",
    "        pipelines.append(current_pipeline)\n",
    "        \n",
    "    return pipelines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# list_numerical_pipelines contains all pipelines for all numerical features. To see the correspondance between the array and the\n",
    "# corresponding feature, get the column names of X_train_numerical df\n",
    "train_numerical = features_train.iloc[:,3:31]\n",
    "list_numerical_pipelines = numerical_pipeline_creator(train_numerical)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_names = train_numerical.columns.tolist()\n",
    "for feature_name in feature_names: \n",
    "    print(feature_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Pipelines according to feature name\n",
    "cant_palabras= list_numerical_pipelines[0]\n",
    "cant_palabras_unicas= list_numerical_pipelines[1]\n",
    "cant_caracteres= list_numerical_pipelines[2]\n",
    "cant_stopwords= list_numerical_pipelines[3]\n",
    "cant_puntuacion= list_numerical_pipelines[4]\n",
    "cant_capitalize= list_numerical_pipelines[5]\n",
    "cant_mayusculas= list_numerical_pipelines[6]\n",
    "promedio_len_word= list_numerical_pipelines[7]\n",
    "cant_caracteres_especiales= list_numerical_pipelines[8]\n",
    "cant_palabras_binned= list_numerical_pipelines[9]\n",
    "cant_palabras_unicas_binned= list_numerical_pipelines[10]\n",
    "cant_caracteres_binned= list_numerical_pipelines[11]\n",
    "cant_stopwords_binned= list_numerical_pipelines[12]\n",
    "cant_puntuacion_binned= list_numerical_pipelines[13]\n",
    "cant_capitalize_binned= list_numerical_pipelines[14]\n",
    "cant_mayusculas_binned= list_numerical_pipelines[15]\n",
    "cant_silabas_binned= list_numerical_pipelines[16]\n",
    "cant_caracteres_especiales_binned= list_numerical_pipelines[17]\n",
    "text_contains_keyword= list_numerical_pipelines[18]\n",
    "has_keyword= list_numerical_pipelines[19]\n",
    "keywords_quantity= list_numerical_pipelines[20]\n",
    "keywords_mean= list_numerical_pipelines[21]\n",
    "keyword_is_hashtag= list_numerical_pipelines[22]\n",
    "keyword_frequency= list_numerical_pipelines[23]\n",
    "cant_hashtags= list_numerical_pipelines[24]\n",
    "has_hashtag= list_numerical_pipelines[25]\n",
    "links_cant= list_numerical_pipelines[26]\n",
    "cant_failed_links= list_numerical_pipelines[27]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3) MODELOS"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3) a) Use of FeatureUnion to put features together"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ATTENTION: Each line is a new feature, so if you wanna add a new feature to the analysis, simply remove the hashtag and run it\n",
    "# again.\n",
    "# If new feature which were not created are to be added, create first and then add it to this group in a new line\n",
    "feats = FeatureUnion([               #   ('cant_palabras', cant_palabras), \n",
    "                                     #   ('cant_palabras_unicas', cant_palabras_unicas),\n",
    "                                     #   ('cant_caracteres', cant_caracteres),\n",
    "                                     #   ('cant_stopwords', cant_stopwords), # parece que overfittea\n",
    "                                     #   ('cant_puntuacion', cant_puntuacion),\n",
    "                                     #   ('cant_capitalize', cant_capitalize),\n",
    "                                     #   ('cant_mayusculas',cant_mayusculas), #parece que overfittea\n",
    "                                         ('promedio_len_word',promedio_len_word),\n",
    "                                     #   ('cant_caracteres_especiales',cant_caracteres_especiales),\n",
    "                                     #   ('cant_palabras_binned',cant_palabras_binned),\n",
    "                                     #   ('cant_palabras_unicas_binned',cant_palabras_unicas_binned),\n",
    "                                     #   ('cant_caracteres_binned',cant_caracteres_binned),\n",
    "                                     #   ('cant_stopwords_binned',cant_stopwords_binned),\n",
    "                                     #   ('cant_puntuacion_binned',cant_puntuacion_binned),\n",
    "                                     #   ('cant_capitalize_binned',cant_capitalize_binned),\n",
    "                                     #   ('cant_mayusculas_binned',cant_mayusculas_binned),\n",
    "                                     #   ('cant_silabas_binned',cant_silabas_binned),\n",
    "                                     #   ('cant_caracteres_especiales_binned',cant_caracteres_especiales_binned),\n",
    "                                     #   ('text_contains_keyword',text_contains_keyword),\n",
    "                                     #   ('has_keyword',has_keyword),\n",
    "                                     #   ('keywords_quantity',keywords_quantity),\n",
    "                                     #   ('keywords_mean',keywords_mean),\n",
    "                                     #   ('keyword_is_hashtag',keyword_is_hashtag),\n",
    "                                     #   ('keyword_frequency',keyword_frequency),\n",
    "                                     #   ('cant_hashtags',cant_hashtags),\n",
    "                                     #   ('has_hashtag',has_hashtag),\n",
    "                                     #   ('links_cant',links_cant),\n",
    "                                     #   ('cant_failed_links',cant_failed_links),\n",
    "                                     #   ('text_pipeline',text_pipeline)\n",
    "                                     #   ('bow', bow_pipeline),\n",
    "                                        ('bow2', bow2_pipeline),\n",
    "                                        ('tfidf', tfidf_pipeline),\n",
    "                                     #   ('tfidf2', tfidf2_pipeline),\n",
    "                                     #   ('tfidf3', tfidf3_pipeline),\n",
    "                                     #   ('tfidf4', tfidf4_pipeline),\n",
    "                     ])\n",
    "\n",
    "feats_lr = FeatureUnion([   ('cant_caracteres', cant_caracteres),\n",
    "                            ('cant_capitalize', cant_capitalize),\n",
    "                            ('promedio_len_word',promedio_len_word),\n",
    "                            ('keyword_frequency',keyword_frequency),\n",
    "                            ('bow2', bow2_pipeline)\n",
    "                     ])\n",
    "\n",
    "feats_nb = FeatureUnion([   ('cant_palabras', cant_palabras), \n",
    "                            ('cant_palabras_unicas', cant_palabras_unicas),\n",
    "                            ('cant_caracteres', cant_caracteres),\n",
    "                            ('cant_capitalize', cant_capitalize),\n",
    "                            ('promedio_len_word',promedio_len_word),\n",
    "                            ('cant_caracteres_especiales',cant_caracteres_especiales),\n",
    "                            ('has_keyword',has_keyword),\n",
    "                            ('keyword_frequency',keyword_frequency),\n",
    "                            ('bow2', bow2_pipeline)\n",
    "                     ])\n",
    "\n",
    "feats_rf = FeatureUnion([   ('cant_palabras', cant_palabras), \n",
    "                            ('cant_caracteres', cant_caracteres),\n",
    "                            ('cant_stopwords', cant_stopwords),\n",
    "                            ('cant_capitalize', cant_capitalize),\n",
    "                            ('promedio_len_word',promedio_len_word),\n",
    "                            ('text_contains_keyword',text_contains_keyword),\n",
    "                            ('keywords_quantity',keywords_quantity),\n",
    "                            ('keywords_mean',keywords_mean),\n",
    "                            ('keyword_frequency',keyword_frequency),\n",
    "                            ('cant_hashtags',cant_hashtags),\n",
    "                            ('links_cant',links_cant),\n",
    "                            ('bow2', bow2_pipeline),\n",
    "                            ('tfidf', tfidf_pipeline)\n",
    "                     ])\n",
    "\n",
    "feats_sgd_svc = FeatureUnion([  ('cant_palabras_unicas_binned',cant_palabras_unicas_binned),\n",
    "                                ('cant_puntuacion_binned',cant_puntuacion_binned),\n",
    "                                ('cant_caracteres_especiales_binned',cant_caracteres_especiales_binned),\n",
    "                                ('text_contains_keyword',text_contains_keyword),\n",
    "                                ('keyword_is_hashtag',keyword_is_hashtag),\n",
    "                                ('keyword_frequency',keyword_frequency),\n",
    "                                ('has_hashtag',has_hashtag),\n",
    "                                ('cant_failed_links',cant_failed_links),\n",
    "                                ('bow2', bow2_pipeline)\n",
    "                     ])\n",
    "\n",
    "feats_sgd_lr = FeatureUnion([   ('cant_palabras_unicas_binned',cant_palabras_unicas_binned),\n",
    "                                ('cant_puntuacion_binned',cant_puntuacion_binned),\n",
    "                                ('cant_caracteres_especiales_binned',cant_caracteres_especiales_binned),\n",
    "                                ('text_contains_keyword',text_contains_keyword),\n",
    "                                ('keyword_is_hashtag',keyword_is_hashtag),\n",
    "                                ('keyword_frequency',keyword_frequency),\n",
    "                                ('has_hashtag',has_hashtag),\n",
    "                                ('cant_failed_links',cant_failed_links),\n",
    "                                ('bow2', bow2_pipeline),\n",
    "                                ('tfidf', tfidf_pipeline)\n",
    "                     ])\n",
    "\n",
    "\n",
    "feature_processing = Pipeline([('feats', feats)])\n",
    "feature_processed = feature_processing.fit_transform(features_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3) b) Dani's junk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(model, test_features, test_labels):\n",
    "    \n",
    "    predictions = model.predict(test_features)\n",
    "    accuracy = round(accuracy_score(test_labels,predictions)*100)\n",
    "    \n",
    "    print(\"\\nMODEL\")\n",
    "    \n",
    "    print(model)\n",
    "    print('Accuracy = {:0.2f}%.'.format(accuracy))\n",
    "    \n",
    "    print()\n",
    "    return accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This funtion saves the value of the hyperparameters and the score of the validation train set \n",
    "def save_prediction_accuracy(modelo,x_train,y_train,x_test,y_test):\n",
    "    clf = modelo\n",
    "    clf.fit(x_train, y_train)\n",
    "    predictions = clf.predict(x_test)\n",
    "    with open(\"prediction_history.csv\", \"a\") as myfile:\n",
    "        #round(clf.score(X_train, y_train)*100)\n",
    "        train_prediction = str((clf.score(X_train, y_train)*100))\n",
    "        print('Score para x_train: '+ train_prediction)\n",
    "        test_prediction = str((accuracy_score(y_test,predictions)*100))\n",
    "        print('Score para x_test: '+ test_prediction)\n",
    "        params = str(modelo)\n",
    "        print('Hiperparametros: '+ str(modelo))\n",
    "        myfile.write(params+','+test_prediction+\",\"+train_prediction+\",\"+str(datetime.datetime.now())+\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_prediction(model, test, name):\n",
    "    predicted = model.predict(test)\n",
    "    sample_submission.target = predicted\n",
    "    sample_submission.to_csv('predictions/'+name+'.csv', index=None)\n",
    "    return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sample_submission = pd.read_csv('original_data/sample_submission.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = features_train.target\n",
    "X = features_train.drop(columns=['target','id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#X_train, X_test, y_train, y_test = train_test_split(train_vec,y,test_size=0.2,random_state=2020)\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=2020)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4) Estimación sobre modelos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_cm(cm, labels, hide_zeroes=False, hide_diagonal=False, hide_threshold=None):\n",
    "    \"\"\"pretty print for confusion matrixes\"\"\"\n",
    "    columnwidth = max([len(x) for x in labels] + [5])  # 5 is value length\n",
    "    empty_cell = \" \" * columnwidth\n",
    "    # Print header\n",
    "    print(\"    \" + empty_cell, end=\" \")\n",
    "    for label in labels:\n",
    "        print(\"%{0}s\".format(columnwidth) % label, end=\" \")\n",
    "    print()\n",
    "    # Print rows\n",
    "    for i, label1 in enumerate(labels):\n",
    "        print(\"    %{0}s\".format(columnwidth) % label1, end=\" \")\n",
    "        for j in range(len(labels)):\n",
    "            cell = \"%{0}.1f\".format(columnwidth) % cm[i, j]\n",
    "            if hide_zeroes:\n",
    "                cell = cell if float(cm[i, j]) != 0 else empty_cell\n",
    "            if hide_diagonal:\n",
    "                cell = cell if i != j else empty_cell\n",
    "            if hide_threshold:\n",
    "                cell = cell if cm[i, j] > hide_threshold else empty_cell\n",
    "            print(cell, end=\" \")\n",
    "        print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fit_and_predict(model,X_train,y_train,X_test,y_test):    \n",
    "    clf = model\n",
    "    clf.fit(X_train, y_train)\n",
    "    predictions=clf.predict(X_test)\n",
    "    confusion_matrix(y_test,predictions)\n",
    "    conf = metrics.confusion_matrix(y_test, predictions)\n",
    "    conf = conf.astype('float') / conf.sum(axis=1)[:, np.newaxis]\n",
    "    print_cm(conf, ['true','false'])\n",
    "    print('-'*50)\n",
    "    print(classification_report(y_test,predictions))\n",
    "    print('-'*50)\n",
    "    print(\"{}\" .format(model))\n",
    "    print('-'*50)\n",
    "    print('Accuracy of classifier on training set:{}%'.format(round(clf.score(X_train, y_train)*100)))\n",
    "    print('-'*50)\n",
    "    print('Accuracy of classifier on test set:{}%' .format(round(accuracy_score(y_test,predictions)*100)))\n",
    "    print('*'*50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This function applies different ESTIMATORS -with specific or default hyperparameters- to a set of features\n",
    "def runClassifiers(X_train, y_train, X_test,y_test):\n",
    "    classifiers = [LogisticRegression(),MultinomialNB(alpha=1),DecisionTreeClassifier(),RandomForestClassifier(),\n",
    "                   SGDClassifier(loss='hinge', penalty='l2', alpha=0.001, random_state=42, max_iter=5, tol=None),\n",
    "                   SGDClassifier(loss='log', penalty='l2', alpha=0.001, random_state=42, max_iter=5, tol=None),\n",
    "                   KNeighborsClassifier(n_neighbors=3, weights = 'distance',p = 1)\n",
    "                 ]\n",
    "\n",
    "    classifier_names = ['Logistic Regression','MultinomialNB','Decision-Tree Classifier',\n",
    "                       'Random-Forest Classifier','SGDC - svc - Classifier','SGDC - lr - Classifier','K-Neighbors Classifier']\n",
    "    features = [feats_lr, feats_nb, feats, feats_rf, feats_sgd_svc, feats_sgd_lr, feats]\n",
    "    \n",
    "    i=-1\n",
    "    for classifier in classifiers:\n",
    "        i = i + 1\n",
    "        clf = Pipeline([\n",
    "            ('features', features[i]),\n",
    "            ('actualClassifier', classifier )\n",
    "        ])\n",
    "        print(classifier_names[i])\n",
    "        fit_and_predict(clf,X_train,y_train,X_test,y_test)\n",
    "        #clf.fit(X_train, y_train)\n",
    "        #docs_test = X_test\n",
    "        #predicted = clf.predict(docs_test)\n",
    "        #print(classifier_names[i],np.mean(predicted == y_test))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "#probando de todo\n",
    "runClassifiers(X_train,y_train,X_test,y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Estimación con Cross Validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "n=5\n",
    "seed=2020\n",
    "kf = KFold(n_splits=n, random_state=seed, shuffle=True)\n",
    " \n",
    "def acc_cross_val(classifier, feats, X):\n",
    "    # Initialize the accuracy of the models to blank list. The accuracy of each model will be appended to this list\n",
    "    accuracy_model = []\n",
    "    # Iterate over each train-test split\n",
    "    for train_index, test_index in kf.split(X):\n",
    "        # Split train-test\n",
    "        X_train, X_test = X.iloc[train_index], X.iloc[test_index]\n",
    "        y_train, y_test = y[train_index], y[test_index]\n",
    "        # Train the model\n",
    "        clf = Pipeline([\n",
    "            ('features', feats),\n",
    "            ('actualClassifier', classifier)\n",
    "        ])\n",
    "        model = clf.fit(X_train, y_train)\n",
    "        # Append to accuracy_model the accuracy of the model\n",
    "        accuracy_model.append(accuracy_score(y_test, model.predict(X_test), normalize=True)*100)\n",
    "\n",
    "    # Print the accuracy    \n",
    "    print(accuracy_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "classifiers = [LogisticRegression(),MultinomialNB(alpha=1),DecisionTreeClassifier(),RandomForestClassifier(),\n",
    "                   SGDClassifier(loss='hinge', penalty='l2', alpha=0.001, random_state=42, max_iter=5, tol=None),\n",
    "                   SGDClassifier(loss='log', penalty='l2', alpha=0.001, random_state=42, max_iter=5, tol=None),\n",
    "                   KNeighborsClassifier(n_neighbors=7, weights = 'distance',p = 1)\n",
    "                 ]\n",
    "\n",
    "classifier_names = ['Logistic Regression','MultinomialNB','Decision-Tree Classifier',\n",
    "                       'Random-Forest Classifier','SGDC - svc - Classifier','SGDC - lr - Classifier','K-Neighbors Classifier']\n",
    "features = [feats_lr, feats_nb, feats, feats_rf, feats_sgd_svc, feats_sgd_lr, feats]\n",
    "i = -1\n",
    "for classifier in classifiers:\n",
    "    i = i+1\n",
    "    print(classifier_names[i])\n",
    "    acc_cross_val(classifier, features[i], X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Multinomial Naive Bayes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_nb = Pipeline([\n",
    "    ('features', feats_nb),\n",
    "    ('multinomialNB', MultinomialNB(alpha=1))\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MultinomialNB(alpha=1, class_prior=None, fit_prior=True)"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "save_prediction_accuracy(model_nb,X_train,y_train,X_test,y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_nb.fit(X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = 'models/nb_model.sav'\n",
    "pickle.dump(model_nb, open(filename, 'wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "                   intercept_scaling=1, l1_ratio=None, max_iter=100,\n",
       "                   multi_class='auto', n_jobs=None, penalty='l2',\n",
       "                   random_state=None, solver='lbfgs', tol=0.0001, verbose=0,\n",
       "                   warm_start=False)"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#model_lr = LogisticRegression()\n",
    "model_lr = Pipeline([\n",
    "    ('features', feats_lr),\n",
    "    ('logisticReg', LogisticRegression())\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model Performance\n",
      "Accuracy = 96.00%.\n"
     ]
    }
   ],
   "source": [
    "save_prediction_accuracy(model_lr,X_train,y_train,X_test,y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_lr.fit(X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = 'models/lr_model.sav'\n",
    "pickle.dump(model_lr, open(filename, 'wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Decision Tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_dt = Pipeline([\n",
    "    ('features', feats),\n",
    "    ('decisionTree', DecisionTreeClassifier())\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_prediction_accuracy(model_dt,X_train,y_train,X_test,y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_dt.fit(X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = 'models/dt_model.sav'\n",
    "pickle.dump(model_dt, open(filename, 'wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## KNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "knn = KNeighborsClassifier(n_neighbors=3, weights = 'distance',p = 1)\n",
    "model_knn = Pipeline([\n",
    "    ('features', feats),\n",
    "    ('KNN', knn)\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_prediction_accuracy(model_knn,X_train,y_train,X_test,y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_knn.fit(X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = 'models/knn_model.sav'\n",
    "pickle.dump(model_knn, open(filename, 'wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MultiLayer Perceptron"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# muy lento\n",
    "'''mlp = MLPClassifier()\n",
    "model_mlp = Pipeline([\n",
    "    ('features', feats),\n",
    "    ('MLP', mlp)\n",
    "])'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#save_prediction_accuracy(model_mlp,X_train,y_train,X_test,y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#model_mlp.fit(X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#filename = 'models/mlp_model.sav'\n",
    "#pickle.dump(model_mlp, open(filename, 'wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SGD (Stochastic Gradient Descent)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_sgd_svc = Pipeline([\n",
    "    ('features', feats_sgd_svc),\n",
    "    ('sgd', SGDClassifier(loss='hinge', penalty='l2', alpha=0.001, random_state=42, max_iter=5, tol=None))\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_prediction_accuracy(model_sgd_svc,X_train,y_train,X_test,y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_sgd_svc.fit(X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = 'models/sgd_model_svc.sav'\n",
    "pickle.dump(model_sgd_svc, open(filename, 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_sgd_lr = Pipeline([\n",
    "    ('features', feats_sgd_lr),\n",
    "    ('sgd', SGDClassifier(loss='log', penalty='l2', alpha=0.001, random_state=42, max_iter=5, tol=None))\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_prediction_accuracy(model_sgd_lr,X_train,y_train,X_test,y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_sgd_lr.fit(X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = 'models/sgd_model_lr.sav'\n",
    "pickle.dump(model_sgd_lr, open(filename, 'wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Ensambles"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Voting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import VotingClassifier\n",
    "\n",
    "# 1ro con todos los modelos\n",
    "vote  = VotingClassifier(estimators=[('lr', model_lr), ('nb', model_nb), ('sgd_lr', model_sgd_lr), ('sgd_svc', model_sgd_svc)], \n",
    "                          voting='hard')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_prediction_accuracy(vote,X_train,y_train,X_test,y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stacking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sclf = StackingClassifier(classifiers=[model_nb,model_lr],\n",
    "                          meta_classifier=LogisticRegression()) #,model_mlp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_prediction_accuracy(sclf,X_train,y_train,X_test,y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Boosting"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### AdaBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "ada = AdaBoostClassifier(random_state = 1, n_estimators=100)\n",
    "\n",
    "model_ada = Pipeline([\n",
    "    ('features', feats),\n",
    "    ('adaboost', ada)\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_prediction_accuracy(model_ada,X_train,y_train,X_test,y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_xgb = Pipeline([\n",
    "    ('features', feats),\n",
    "    ('xgboost', XGBClassifier())\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_prediction_accuracy(model_xgb,X_train,y_train,X_test,y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LightGBM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_lgbm = Pipeline([\n",
    "    ('features', feats),\n",
    "    ('lgbm', lgb.LGBMClassifier())\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_prediction_accuracy(model_lgbm,X_train,y_train,X_test,y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Bagging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_rf = Pipeline([\n",
    "    ('features', feats),\n",
    "    ('rf', RandomForestClassifier())\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_prediction_accuracy(model_rf,X_train,y_train,X_test,y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Optimization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hyperparametros"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Funciones de optimizacion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from bayes_opt import BayesianOptimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "def GridSearchCV_models(model_name, model, parameters, X_train, y_train, X_test, y_test, _cv):\n",
    "    \n",
    "    pipe_clf = Pipeline([\n",
    "        ('features', feats),\n",
    "        (model_name, model),\n",
    "    ])\n",
    "        \n",
    "    best_clf = GridSearchCV(estimator = pipe_clf, param_grid = parameters, cv = _cv, verbose=2, n_jobs = -1)\n",
    "    best_clf.fit(X_train, y_train)\n",
    "        \n",
    "    evaluate(best_clf, X_test, y_test)\n",
    "            \n",
    "    return best_clf.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "def RandomizedSearchCV_models(model_name, model, parameters, X_train, y_train, X_test, y_test, _n_iter, _cv):\n",
    "    \n",
    "    pipe_clf = Pipeline([\n",
    "        ('features', feats),\n",
    "        (model_name, model),\n",
    "    ])\n",
    "        \n",
    "    best_clf = RandomizedSearchCV(estimator = pipe_clf, param_distributions = parameters, n_iter = _n_iter, cv = _cv, verbose=2, random_state=42, n_jobs = -1)\n",
    "    best_clf.fit(X_train, y_train)\n",
    "        \n",
    "    evaluate(best_clf, X_test, y_test)\n",
    "            \n",
    "    return best_clf.best_params_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Algoritmos optimizados"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logistic Regression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** RandomizedSearchCV **"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = \"LR\"\n",
    "model = LogisticRegression()\n",
    "parameters = {\n",
    "    'LR__penalty':['l1', 'l2', 'elasticnet','none'],\n",
    "    'LR__C': [0.1, 0.5, 0.25, 0.75, 1, 2, 3, 2.5, 1.5],\n",
    "    'LR__solver':[ 'lbfgs', 'liblinear', 'sag', 'saga'],\n",
    "    'LR__multi_class': ['auto', 'ovr', 'multinomial'],\n",
    "    'LR__max_iter': np.arange(100,2000,200),\n",
    "    \"LR__n_jobs\": [-1]\n",
    "}\n",
    "_n_iter = 10\n",
    "_cv = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 10 candidates, totalling 50 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  33 tasks      | elapsed:  1.9min\n",
      "[Parallel(n_jobs=-1)]: Done  50 out of  50 | elapsed:  2.0min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "MODEL\n",
      "RandomizedSearchCV(cv=5, error_score=nan,\n",
      "                   estimator=Pipeline(memory=None,\n",
      "                                      steps=[('features',\n",
      "                                              FeatureUnion(n_jobs=None,\n",
      "                                                           transformer_list=[('cant_palabras',\n",
      "                                                                              Pipeline(memory=None,\n",
      "                                                                                       steps=[('selector',\n",
      "                                                                                               NumberSelector(key='#palabras'))],\n",
      "                                                                                       verbose=False)),\n",
      "                                                                             ('links_cant',\n",
      "                                                                              Pipeline(memory=None,\n",
      "                                                                                       steps=[('selector',\n",
      "                                                                                               NumberSelector(key='links_cant'))],\n",
      "                                                                                       verbose=False)),\n",
      "                                                                             ('bow...\n",
      "                   param_distributions={'LR__C': [0.1, 0.5, 0.25, 0.75, 1, 2, 5,\n",
      "                                                  10, 20, 30, 50],\n",
      "                                        'LR__max_iter': array([ 500, 1000, 1500, 2000]),\n",
      "                                        'LR__multi_class': ['auto', 'ovr',\n",
      "                                                            'multinomial'],\n",
      "                                        'LR__n_jobs': [-1],\n",
      "                                        'LR__penalty': ['l1', 'l2',\n",
      "                                                        'elasticnet', 'none'],\n",
      "                                        'LR__solver': ['lbfgs', 'liblinear',\n",
      "                                                       'sag', 'saga']},\n",
      "                   pre_dispatch='2*n_jobs', random_state=42, refit=True,\n",
      "                   return_train_score=False, scoring=None, verbose=2)\n",
      "Accuracy = 81.00%.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# RandomizedSearchCV\n",
    "\n",
    "best_params_LR_random = RandomizedSearchCV_models(model_name, model, parameters, X_train.fillna(0), y_train, X_test.fillna(0), y_test, _n_iter, _cv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'LR__solver': 'saga', 'LR__penalty': 'l2', 'LR__n_jobs': -1, 'LR__multi_class': 'multinomial', 'LR__max_iter': 1500, 'LR__C': 2}\n"
     ]
    }
   ],
   "source": [
    "print(best_params_LR_random)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** GridSearchCV_models **"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = \"LR\"\n",
    "model = LogisticRegression()\n",
    "parameters = {\n",
    "    'LR__penalty':['l2'],\n",
    "    'LR__C': [0.1,0.5,0.25,0.75,1,2,5,10],\n",
    "    'LR__solver':[ 'lbfgs', 'liblinear', 'sag', 'saga'],\n",
    "    'LR__multi_class': ['multinomial'],\n",
    "    'LR__max_iter': np.arange(100,1500,100),\n",
    "    \"LR__n_jobs\": [-1]\n",
    "}\n",
    "_n_iter = 5\n",
    "_cv = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_params_LR_grid = GridSearchCV_models(model_name, model, parameters, X_train.fillna(0), y_train, X_test.fillna(0), y_test, _cv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(best_params_LR_grid)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** Bayesian Optimization **"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [],
   "source": [
    "LR_parameters = {\n",
    "    'C': [0.01,10],\n",
    "    'max_iter':(100,2000),\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [],
   "source": [
    "def LR_eval( C, max_iter):\n",
    "  \n",
    "    model = LogisticRegression(\n",
    "        penalty = \"l2\",\n",
    "        C = C,\n",
    "        solver = \"saga\",\n",
    "        multi_class = \"multinomial\",\n",
    "        max_iter = max_iter,\n",
    "    )               \n",
    "    \n",
    "    pipe_clf = Pipeline([\n",
    "        ('features', feats),\n",
    "        (\"LR\", model),\n",
    "    ])\n",
    "                                 \n",
    "    pipe_clf.fit(X_train.fillna(0),y_train)\n",
    "                   \n",
    "    return (accuracy_score(y_test, pipe_clf.predict(X_test.fillna(0)))*100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "lrBO = BayesianOptimization(LR_eval, LR_parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "|   iter    |  target   |     C     | max_iter  |\n",
      "-------------------------------------------------\n",
      "| \u001b[0m 1       \u001b[0m | \u001b[0m 80.83   \u001b[0m | \u001b[0m 0.2668  \u001b[0m | \u001b[0m 1.56e+03\u001b[0m |\n",
      "| \u001b[0m 2       \u001b[0m | \u001b[0m 80.56   \u001b[0m | \u001b[0m 3.751   \u001b[0m | \u001b[0m 1.348e+0\u001b[0m |\n",
      "| \u001b[0m 3       \u001b[0m | \u001b[0m 80.76   \u001b[0m | \u001b[0m 1.874   \u001b[0m | \u001b[0m 1.558e+0\u001b[0m |\n",
      "| \u001b[0m 4       \u001b[0m | \u001b[0m 75.11   \u001b[0m | \u001b[0m 0.01    \u001b[0m | \u001b[0m 1.653e+0\u001b[0m |\n",
      "| \u001b[0m 5       \u001b[0m | \u001b[0m 80.43   \u001b[0m | \u001b[0m 6.0     \u001b[0m | \u001b[0m 1.38e+03\u001b[0m |\n",
      "| \u001b[0m 6       \u001b[0m | \u001b[0m 80.43   \u001b[0m | \u001b[0m 9.882   \u001b[0m | \u001b[0m 1.304e+0\u001b[0m |\n",
      "| \u001b[0m 7       \u001b[0m | \u001b[0m 75.05   \u001b[0m | \u001b[0m 0.01    \u001b[0m | \u001b[0m 1.25e+03\u001b[0m |\n",
      "| \u001b[0m 8       \u001b[0m | \u001b[0m 80.83   \u001b[0m | \u001b[0m 0.849   \u001b[0m | \u001b[0m 1.324e+0\u001b[0m |\n",
      "| \u001b[0m 9       \u001b[0m | \u001b[0m 80.24   \u001b[0m | \u001b[0m 10.0    \u001b[0m | \u001b[0m 1.579e+0\u001b[0m |\n",
      "| \u001b[95m 10      \u001b[0m | \u001b[95m 80.96   \u001b[0m | \u001b[95m 1.012   \u001b[0m | \u001b[95m 1.42e+03\u001b[0m |\n",
      "| \u001b[0m 11      \u001b[0m | \u001b[0m 80.3    \u001b[0m | \u001b[0m 8.42    \u001b[0m | \u001b[0m 1.451e+0\u001b[0m |\n",
      "| \u001b[0m 12      \u001b[0m | \u001b[0m 80.83   \u001b[0m | \u001b[0m 1.223   \u001b[0m | \u001b[0m 1.507e+0\u001b[0m |\n",
      "| \u001b[0m 13      \u001b[0m | \u001b[0m 80.96   \u001b[0m | \u001b[0m 1.677   \u001b[0m | \u001b[0m 194.6   \u001b[0m |\n",
      "| \u001b[0m 14      \u001b[0m | \u001b[0m 76.1    \u001b[0m | \u001b[0m 0.01    \u001b[0m | \u001b[0m 158.3   \u001b[0m |\n",
      "| \u001b[0m 15      \u001b[0m | \u001b[0m 80.96   \u001b[0m | \u001b[0m 1.632   \u001b[0m | \u001b[0m 213.6   \u001b[0m |\n",
      "| \u001b[0m 16      \u001b[0m | \u001b[0m 80.83   \u001b[0m | \u001b[0m 0.2775  \u001b[0m | \u001b[0m 1.483e+0\u001b[0m |\n",
      "| \u001b[0m 17      \u001b[0m | \u001b[0m 80.37   \u001b[0m | \u001b[0m 9.667   \u001b[0m | \u001b[0m 1.402e+0\u001b[0m |\n",
      "| \u001b[0m 18      \u001b[0m | \u001b[0m 80.83   \u001b[0m | \u001b[0m 7.87    \u001b[0m | \u001b[0m 247.2   \u001b[0m |\n",
      "| \u001b[0m 19      \u001b[0m | \u001b[0m 80.96   \u001b[0m | \u001b[0m 1.552   \u001b[0m | \u001b[0m 276.9   \u001b[0m |\n",
      "| \u001b[0m 20      \u001b[0m | \u001b[0m 80.76   \u001b[0m | \u001b[0m 9.777   \u001b[0m | \u001b[0m 306.2   \u001b[0m |\n",
      "| \u001b[0m 21      \u001b[0m | \u001b[0m 80.63   \u001b[0m | \u001b[0m 0.4607  \u001b[0m | \u001b[0m 341.4   \u001b[0m |\n",
      "| \u001b[0m 22      \u001b[0m | \u001b[0m 80.96   \u001b[0m | \u001b[0m 10.0    \u001b[0m | \u001b[0m 379.4   \u001b[0m |\n",
      "=================================================\n"
     ]
    }
   ],
   "source": [
    "lrBO.maximize(init_points=2,n_iter=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'params': {'C': 1.011702931147121, 'max_iter': 1419.6441643789612},\n",
       " 'target': 80.9586342744583}"
      ]
     },
     "execution_count": 169,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lrBO.max"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SGDClassifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**RandomizedSearchCV** "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 311,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = \"SGD\"\n",
    "model = SGDClassifier()\n",
    "parameters = {\n",
    "    'SGD__penalty':['l1', 'l2', 'elasticnet'],\n",
    "    \"SGD__loss\":  ['hinge', 'log', 'modified_huber', 'perceptron'],\n",
    "    'SGD__alpha': [0.1, 0.5, 0.25, 0.75, 1, 0.001,0.01,0.0001,0.15,0.2],\n",
    "    'SGD__epsilon':[0.1, 0.5, 0.25, 0.75, 1, 0.001,0.01,0.0001,0.2,0.15],\n",
    "    'SGD__learning_rate': [\"optimal\",\"constant\",\"invscaling\",\"adaptive\"],\n",
    "    'SGD__max_iter': np.arange(50,3000,50),\n",
    "    \"SGD__n_jobs\": [-1],\n",
    "    \"SGD__eta0\":[0.1,0.001,1,0.0001,0.00001,1]\n",
    "    #\"SGD__average\": [True, False],\n",
    "    #\"SGD__early_stopping\": [True, False],\n",
    "    #\"SGD__fit_intercept\": [True, False],\n",
    "    #\"SGD__shuffle\": [True, False]\n",
    "}\n",
    "\n",
    "_n_iter = 15\n",
    "_cv = 15"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 312,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 15 folds for each of 15 candidates, totalling 225 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  33 tasks      | elapsed:    9.5s\n",
      "[Parallel(n_jobs=-1)]: Done 154 tasks      | elapsed:   35.5s\n",
      "[Parallel(n_jobs=-1)]: Done 225 out of 225 | elapsed:   49.2s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "MODEL\n",
      "RandomizedSearchCV(cv=15, error_score=nan,\n",
      "                   estimator=Pipeline(memory=None,\n",
      "                                      steps=[('features',\n",
      "                                              FeatureUnion(n_jobs=None,\n",
      "                                                           transformer_list=[('cant_palabras',\n",
      "                                                                              Pipeline(memory=None,\n",
      "                                                                                       steps=[('selector',\n",
      "                                                                                               NumberSelector(key='#palabras'))],\n",
      "                                                                                       verbose=False)),\n",
      "                                                                             ('links_cant',\n",
      "                                                                              Pipeline(memory=None,\n",
      "                                                                                       steps=[('selector',\n",
      "                                                                                               NumberSelector(key='links_cant'))],\n",
      "                                                                                       verbose=False)),\n",
      "                                                                             ('bo...\n",
      "        600,  650,  700,  750,  800,  850,  900,  950, 1000, 1050, 1100,\n",
      "       1150, 1200, 1250, 1300, 1350, 1400, 1450, 1500, 1550, 1600, 1650,\n",
      "       1700, 1750, 1800, 1850, 1900, 1950, 2000, 2050, 2100, 2150, 2200,\n",
      "       2250, 2300, 2350, 2400, 2450, 2500, 2550, 2600, 2650, 2700, 2750,\n",
      "       2800, 2850, 2900, 2950]),\n",
      "                                        'SGD__n_jobs': [-1],\n",
      "                                        'SGD__penalty': ['l1', 'l2',\n",
      "                                                         'elasticnet']},\n",
      "                   pre_dispatch='2*n_jobs', random_state=42, refit=True,\n",
      "                   return_train_score=False, scoring=None, verbose=2)\n",
      "Accuracy = 80.00%.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# RandomizedSearchCV\n",
    "\n",
    "best_params_SGD_random = RandomizedSearchCV_models(model_name, model, parameters, X_train.fillna(0), y_train, X_test.fillna(0), y_test, _n_iter, _cv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 313,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'SGD__penalty': 'elasticnet', 'SGD__n_jobs': -1, 'SGD__max_iter': 300, 'SGD__loss': 'log', 'SGD__learning_rate': 'optimal', 'SGD__eta0': 1, 'SGD__epsilon': 0.2, 'SGD__alpha': 0.001}\n"
     ]
    }
   ],
   "source": [
    "print(best_params_SGD_random)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 317,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = \"SGD\"\n",
    "model = SGDClassifier()\n",
    "parameters = {\n",
    "    'SGD__penalty':[ 'elasticnet'],\n",
    "    \"SGD__loss\":  [ 'modified_huber'],\n",
    "    'SGD__alpha': [ 0.001, 0.005, 0.002,0.0008],\n",
    "    'SGD__epsilon':[ 0.01, 0.05, 0.02,0.008],\n",
    "    'SGD__learning_rate': [\"adaptive\"],\n",
    "    'SGD__max_iter': np.arange(800,1500,100),\n",
    "    \"SGD__n_jobs\": [-1],\n",
    "    \"SGD__eta0\":[0.1,0.02,0.05,0.5,0.2],\n",
    "    #\"SGD__average\": [True, False],\n",
    "    #\"SGD__early_stopping\": [True, False],\n",
    "    #\"SGD__fit_intercept\": [True, False],\n",
    "    #\"SGD__shuffle\": [True, False]\n",
    "}\n",
    "\n",
    "_n_iter = 15\n",
    "_cv = 15"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 318,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 15 folds for each of 15 candidates, totalling 225 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  33 tasks      | elapsed:   12.0s\n",
      "[Parallel(n_jobs=-1)]: Done 154 tasks      | elapsed:   50.2s\n",
      "[Parallel(n_jobs=-1)]: Done 225 out of 225 | elapsed:  1.2min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "MODEL\n",
      "RandomizedSearchCV(cv=15, error_score=nan,\n",
      "                   estimator=Pipeline(memory=None,\n",
      "                                      steps=[('features',\n",
      "                                              FeatureUnion(n_jobs=None,\n",
      "                                                           transformer_list=[('cant_palabras',\n",
      "                                                                              Pipeline(memory=None,\n",
      "                                                                                       steps=[('selector',\n",
      "                                                                                               NumberSelector(key='#palabras'))],\n",
      "                                                                                       verbose=False)),\n",
      "                                                                             ('links_cant',\n",
      "                                                                              Pipeline(memory=None,\n",
      "                                                                                       steps=[('selector',\n",
      "                                                                                               NumberSelector(key='links_cant'))],\n",
      "                                                                                       verbose=False)),\n",
      "                                                                             ('bo...\n",
      "                                                       0.0008],\n",
      "                                        'SGD__epsilon': [0.01, 0.05, 0.02,\n",
      "                                                         0.008],\n",
      "                                        'SGD__eta0': [0.1, 0.02, 0.05, 0.5,\n",
      "                                                      0.2],\n",
      "                                        'SGD__learning_rate': ['adaptive'],\n",
      "                                        'SGD__loss': ['modified_huber'],\n",
      "                                        'SGD__max_iter': array([ 800,  900, 1000, 1100, 1200, 1300, 1400]),\n",
      "                                        'SGD__n_jobs': [-1],\n",
      "                                        'SGD__penalty': ['elasticnet']},\n",
      "                   pre_dispatch='2*n_jobs', random_state=42, refit=True,\n",
      "                   return_train_score=False, scoring=None, verbose=2)\n",
      "Accuracy = 81.00%.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "best_params_SGD_random = RandomizedSearchCV_models(model_name, model, parameters, X_train.fillna(0), y_train, X_test.fillna(0), y_test, _n_iter, _cv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 319,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'SGD__penalty': 'elasticnet', 'SGD__n_jobs': -1, 'SGD__max_iter': 1000, 'SGD__loss': 'modified_huber', 'SGD__learning_rate': 'adaptive', 'SGD__eta0': 0.05, 'SGD__epsilon': 0.01, 'SGD__alpha': 0.002}\n"
     ]
    }
   ],
   "source": [
    "print(best_params_SGD_random)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**GridSearchCV_models**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 320,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = \"SGD\"\n",
    "model = SGDClassifier()\n",
    "parameters = {\n",
    "    'SGD__penalty':[ 'elasticnet'],\n",
    "    \"SGD__loss\":  [ 'modified_huber'],\n",
    "    'SGD__alpha': [ 0.001, 0.005, 0.002,0.0008],\n",
    "    'SGD__epsilon':[ 0.01, 0.05, 0.02,0.008],\n",
    "    'SGD__learning_rate': [\"adaptive\"],\n",
    "    'SGD__max_iter': np.arange(800,1500,100),\n",
    "    \"SGD__n_jobs\": [-1],\n",
    "    \"SGD__eta0\":[0.1,0.02,0.05,0.5,0.2],\n",
    "    \"SGD__average\": [True, False],\n",
    "    \"SGD__early_stopping\": [True, False],\n",
    "    \"SGD__fit_intercept\": [True, False],\n",
    "    \"SGD__shuffle\": [True, False]\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 8960 candidates, totalling 26880 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  33 tasks      | elapsed:    7.6s\n",
      "[Parallel(n_jobs=-1)]: Done 154 tasks      | elapsed:   31.6s\n",
      "[Parallel(n_jobs=-1)]: Done 357 tasks      | elapsed:  1.2min\n",
      "[Parallel(n_jobs=-1)]: Done 640 tasks      | elapsed:  2.2min\n",
      "[Parallel(n_jobs=-1)]: Done 1005 tasks      | elapsed:  3.5min\n",
      "[Parallel(n_jobs=-1)]: Done 1450 tasks      | elapsed:  5.1min\n",
      "[Parallel(n_jobs=-1)]: Done 1977 tasks      | elapsed:  7.1min\n",
      "[Parallel(n_jobs=-1)]: Done 2584 tasks      | elapsed:  9.9min\n",
      "[Parallel(n_jobs=-1)]: Done 3273 tasks      | elapsed: 13.0min\n"
     ]
    }
   ],
   "source": [
    "best_params_SGD_grid = GridSearchCV_models(model_name, model, parameters, X_train.fillna(0), y_train, X_test.fillna(0), y_test, 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(best_params_SGD_random)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = \"SGD\"\n",
    "model = SGDClassifier()\n",
    "parameters = {\n",
    "    'SGD__penalty':[ 'elasticnet'],\n",
    "    \"SGD__loss\":  [ 'modified_huber'],\n",
    "    'SGD__alpha': [ 0.001, 0.005, 0.002,0.0008],\n",
    "    'SGD__epsilon':[ 0.01, 0.05, 0.02,0.008],\n",
    "    'SGD__learning_rate': [\"adaptive\"],\n",
    "    'SGD__max_iter': np.arange(800,1500,100),\n",
    "    \"SGD__n_jobs\": [-1],\n",
    "    \"SGD__eta0\":[0.1,0.02,0.05,0.5,0.2],\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 295,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 448 candidates, totalling 1344 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  33 tasks      | elapsed:    7.2s\n",
      "[Parallel(n_jobs=-1)]: Done 154 tasks      | elapsed:   33.4s\n",
      "[Parallel(n_jobs=-1)]: Done 357 tasks      | elapsed:  1.4min\n",
      "[Parallel(n_jobs=-1)]: Done 640 tasks      | elapsed:  2.4min\n",
      "[Parallel(n_jobs=-1)]: Done 1005 tasks      | elapsed:  3.7min\n",
      "[Parallel(n_jobs=-1)]: Done 1344 out of 1344 | elapsed:  5.0min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "MODEL\n",
      "GridSearchCV(cv=3, error_score=nan,\n",
      "             estimator=Pipeline(memory=None,\n",
      "                                steps=[('features',\n",
      "                                        FeatureUnion(n_jobs=None,\n",
      "                                                     transformer_list=[('cant_palabras',\n",
      "                                                                        Pipeline(memory=None,\n",
      "                                                                                 steps=[('selector',\n",
      "                                                                                         NumberSelector(key='#palabras'))],\n",
      "                                                                                 verbose=False)),\n",
      "                                                                       ('links_cant',\n",
      "                                                                        Pipeline(memory=None,\n",
      "                                                                                 steps=[('selector',\n",
      "                                                                                         NumberSelector(key='links_cant'))],\n",
      "                                                                                 verbose=False)),\n",
      "                                                                       ('bow',\n",
      "                                                                        Pipe...\n",
      "             param_grid={'SGD__alpha': [0.001, 0.005, 0.002, 0.0008],\n",
      "                         'SGD__epsilon': [0.01, 0.05, 0.02, 0.008],\n",
      "                         'SGD__eta0': [0.1, 0.02, 0.05, 0.5],\n",
      "                         'SGD__learning_rate': ['adaptive'],\n",
      "                         'SGD__loss': ['modified_huber'],\n",
      "                         'SGD__max_iter': array([ 800,  900, 1000, 1100, 1200, 1300, 1400]),\n",
      "                         'SGD__n_jobs': [-1], 'SGD__penalty': ['elasticnet']},\n",
      "             pre_dispatch='2*n_jobs', refit=True, return_train_score=False,\n",
      "             scoring=None, verbose=2)\n",
      "Accuracy = 82.00%.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "best_params_SGD_grid = GridSearchCV_models(model_name, model, parameters, X_train.fillna(0), y_train, X_test.fillna(0), y_test, 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 298,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'SGD__penalty': 'elasticnet', 'SGD__n_jobs': -1, 'SGD__max_iter': 1000, 'SGD__loss': 'modified_huber', 'SGD__learning_rate': 'adaptive', 'SGD__eta0': 0.05, 'SGD__epsilon': 0.02, 'SGD__alpha': 0.002}\n"
     ]
    }
   ],
   "source": [
    "print(best_params_SGD_random)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "best = {'SGD__penalty': 'elasticnet', 'SGD__n_jobs': -1, 'SGD__max_iter': 1000, 'SGD__loss': 'modified_huber', 'SGD__learning_rate': 'adaptive', 'SGD__eta0': 0.05, 'SGD__epsilon': 0.02, 'SGD__alpha': 0.002}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** Bayesian Optimization **"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [],
   "source": [
    "SGD_parameters = {\n",
    "    'epsilon': (0.0001,0.5),\n",
    "    'alpha': (0.0001,0.5),\n",
    "    \"eta0\": (0.001,0.5),\n",
    "    \"max_iter\": (100,1500)\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [],
   "source": [
    "def SGD_eval(alpha, epsilon, eta0, max_iter):\n",
    "  \n",
    "    model = SGDClassifier(\n",
    "        \n",
    "    penalty = 'elasticnet',\n",
    "    loss ='modified_huber',\n",
    "    learning_rate= \"adaptive\",\n",
    "    n_jobs=-1,\n",
    "    \n",
    "    alpha= alpha,\n",
    "    epsilon = epsilon,\n",
    "    max_iter= max_iter,\n",
    "    eta0=eta0\n",
    "    )               \n",
    "    \n",
    "    pipe_clf = Pipeline([\n",
    "        ('features', feats),\n",
    "        (\"clf\", model),\n",
    "    ])\n",
    "                                 \n",
    "    pipe_clf.fit(X_train.fillna(0),y_train)\n",
    "                   \n",
    "    return (accuracy_score(y_test, pipe_clf.predict(X_test.fillna(0)))*100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [],
   "source": [
    "SGD_BO = BayesianOptimization(SGD_eval, SGD_parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['alpha', 'epsilon', 'eta0', 'max_iter']\n"
     ]
    }
   ],
   "source": [
    "print(SGD_BO.space.keys)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "|   iter    |  target   |   alpha   |  epsilon  |   eta0    | max_iter  |\n",
      "-------------------------------------------------------------------------\n",
      "| \u001b[0m 1       \u001b[0m | \u001b[0m 57.45   \u001b[0m | \u001b[0m 0.1187  \u001b[0m | \u001b[0m 0.3183  \u001b[0m | \u001b[0m 0.02295 \u001b[0m | \u001b[0m 1.344e+0\u001b[0m |\n",
      "| \u001b[0m 2       \u001b[0m | \u001b[0m 56.27   \u001b[0m | \u001b[0m 0.0641  \u001b[0m | \u001b[0m 0.3518  \u001b[0m | \u001b[0m 0.4925  \u001b[0m | \u001b[0m 574.1   \u001b[0m |\n",
      "| \u001b[0m 3       \u001b[0m | \u001b[0m 55.88   \u001b[0m | \u001b[0m 0.292   \u001b[0m | \u001b[0m 0.2749  \u001b[0m | \u001b[0m 0.3572  \u001b[0m | \u001b[0m 303.8   \u001b[0m |\n",
      "| \u001b[0m 4       \u001b[0m | \u001b[0m 54.63   \u001b[0m | \u001b[0m 0.4997  \u001b[0m | \u001b[0m 0.4398  \u001b[0m | \u001b[0m 0.3856  \u001b[0m | \u001b[0m 1.211e+0\u001b[0m |\n",
      "| \u001b[0m 5       \u001b[0m | \u001b[0m 55.88   \u001b[0m | \u001b[0m 0.3417  \u001b[0m | \u001b[0m 0.106   \u001b[0m | \u001b[0m 0.2487  \u001b[0m | \u001b[0m 1.38e+03\u001b[0m |\n",
      "| \u001b[0m 6       \u001b[0m | \u001b[0m 54.83   \u001b[0m | \u001b[0m 0.4358  \u001b[0m | \u001b[0m 0.02889 \u001b[0m | \u001b[0m 0.1193  \u001b[0m | \u001b[0m 1.343e+0\u001b[0m |\n",
      "| \u001b[0m 7       \u001b[0m | \u001b[0m 55.02   \u001b[0m | \u001b[0m 0.2615  \u001b[0m | \u001b[0m 0.3839  \u001b[0m | \u001b[0m 0.4381  \u001b[0m | \u001b[0m 574.1   \u001b[0m |\n",
      "| \u001b[0m 8       \u001b[0m | \u001b[0m 55.61   \u001b[0m | \u001b[0m 0.2764  \u001b[0m | \u001b[0m 0.08139 \u001b[0m | \u001b[0m 0.367   \u001b[0m | \u001b[0m 846.8   \u001b[0m |\n",
      "| \u001b[0m 9       \u001b[0m | \u001b[0m 56.27   \u001b[0m | \u001b[0m 0.1837  \u001b[0m | \u001b[0m 0.4882  \u001b[0m | \u001b[0m 0.3148  \u001b[0m | \u001b[0m 1.429e+0\u001b[0m |\n",
      "| \u001b[0m 10      \u001b[0m | \u001b[0m 55.22   \u001b[0m | \u001b[0m 0.3945  \u001b[0m | \u001b[0m 0.05175 \u001b[0m | \u001b[0m 0.4156  \u001b[0m | \u001b[0m 458.2   \u001b[0m |\n",
      "| \u001b[0m 11      \u001b[0m | \u001b[0m 55.42   \u001b[0m | \u001b[0m 0.3415  \u001b[0m | \u001b[0m 0.1945  \u001b[0m | \u001b[0m 0.1777  \u001b[0m | \u001b[0m 1.135e+0\u001b[0m |\n",
      "| \u001b[95m 12      \u001b[0m | \u001b[95m 80.83   \u001b[0m | \u001b[95m 0.002686\u001b[0m | \u001b[95m 0.2777  \u001b[0m | \u001b[95m 0.08273 \u001b[0m | \u001b[95m 388.6   \u001b[0m |\n",
      "| \u001b[0m 13      \u001b[0m | \u001b[0m 56.86   \u001b[0m | \u001b[0m 0.1809  \u001b[0m | \u001b[0m 0.341   \u001b[0m | \u001b[0m 0.08158 \u001b[0m | \u001b[0m 388.6   \u001b[0m |\n",
      "| \u001b[0m 14      \u001b[0m | \u001b[0m 61.0    \u001b[0m | \u001b[0m 0.04953 \u001b[0m | \u001b[0m 0.4345  \u001b[0m | \u001b[0m 0.106   \u001b[0m | \u001b[0m 816.1   \u001b[0m |\n",
      "| \u001b[0m 15      \u001b[0m | \u001b[0m 55.88   \u001b[0m | \u001b[0m 0.2187  \u001b[0m | \u001b[0m 0.4252  \u001b[0m | \u001b[0m 0.4982  \u001b[0m | \u001b[0m 656.8   \u001b[0m |\n",
      "| \u001b[0m 16      \u001b[0m | \u001b[0m 54.63   \u001b[0m | \u001b[0m 0.4907  \u001b[0m | \u001b[0m 0.1647  \u001b[0m | \u001b[0m 0.4523  \u001b[0m | \u001b[0m 826.1   \u001b[0m |\n",
      "| \u001b[0m 17      \u001b[0m | \u001b[0m 57.52   \u001b[0m | \u001b[0m 0.04526 \u001b[0m | \u001b[0m 0.343   \u001b[0m | \u001b[0m 0.4606  \u001b[0m | \u001b[0m 447.6   \u001b[0m |\n",
      "| \u001b[0m 18      \u001b[0m | \u001b[0m 55.88   \u001b[0m | \u001b[0m 0.3099  \u001b[0m | \u001b[0m 0.4023  \u001b[0m | \u001b[0m 0.4841  \u001b[0m | \u001b[0m 865.9   \u001b[0m |\n",
      "| \u001b[0m 19      \u001b[0m | \u001b[0m 55.42   \u001b[0m | \u001b[0m 0.3745  \u001b[0m | \u001b[0m 0.142   \u001b[0m | \u001b[0m 0.02693 \u001b[0m | \u001b[0m 1.287e+0\u001b[0m |\n",
      "| \u001b[0m 20      \u001b[0m | \u001b[0m 54.69   \u001b[0m | \u001b[0m 0.4916  \u001b[0m | \u001b[0m 0.1068  \u001b[0m | \u001b[0m 0.2284  \u001b[0m | \u001b[0m 391.4   \u001b[0m |\n",
      "| \u001b[0m 21      \u001b[0m | \u001b[0m 65.07   \u001b[0m | \u001b[0m 0.02836 \u001b[0m | \u001b[0m 0.3291  \u001b[0m | \u001b[0m 0.1354  \u001b[0m | \u001b[0m 720.6   \u001b[0m |\n",
      "| \u001b[0m 22      \u001b[0m | \u001b[0m 55.35   \u001b[0m | \u001b[0m 0.4478  \u001b[0m | \u001b[0m 0.2615  \u001b[0m | \u001b[0m 0.001543\u001b[0m | \u001b[0m 590.4   \u001b[0m |\n",
      "| \u001b[0m 23      \u001b[0m | \u001b[0m 55.48   \u001b[0m | \u001b[0m 0.3951  \u001b[0m | \u001b[0m 0.3065  \u001b[0m | \u001b[0m 0.324   \u001b[0m | \u001b[0m 488.1   \u001b[0m |\n",
      "| \u001b[0m 24      \u001b[0m | \u001b[0m 66.91   \u001b[0m | \u001b[0m 0.03416 \u001b[0m | \u001b[0m 0.01273 \u001b[0m | \u001b[0m 0.07156 \u001b[0m | \u001b[0m 1.226e+0\u001b[0m |\n",
      "| \u001b[0m 25      \u001b[0m | \u001b[0m 55.15   \u001b[0m | \u001b[0m 0.3303  \u001b[0m | \u001b[0m 0.4462  \u001b[0m | \u001b[0m 0.399   \u001b[0m | \u001b[0m 375.6   \u001b[0m |\n",
      "| \u001b[0m 26      \u001b[0m | \u001b[0m 55.42   \u001b[0m | \u001b[0m 0.3047  \u001b[0m | \u001b[0m 0.1308  \u001b[0m | \u001b[0m 0.1165  \u001b[0m | \u001b[0m 457.5   \u001b[0m |\n",
      "| \u001b[0m 27      \u001b[0m | \u001b[0m 54.17   \u001b[0m | \u001b[0m 0.3979  \u001b[0m | \u001b[0m 0.1267  \u001b[0m | \u001b[0m 0.2122  \u001b[0m | \u001b[0m 1.369e+0\u001b[0m |\n",
      "| \u001b[0m 28      \u001b[0m | \u001b[0m 54.76   \u001b[0m | \u001b[0m 0.4875  \u001b[0m | \u001b[0m 0.4495  \u001b[0m | \u001b[0m 0.1401  \u001b[0m | \u001b[0m 950.5   \u001b[0m |\n",
      "| \u001b[0m 29      \u001b[0m | \u001b[0m 55.42   \u001b[0m | \u001b[0m 0.3339  \u001b[0m | \u001b[0m 0.255   \u001b[0m | \u001b[0m 0.2865  \u001b[0m | \u001b[0m 238.8   \u001b[0m |\n",
      "| \u001b[0m 30      \u001b[0m | \u001b[0m 55.48   \u001b[0m | \u001b[0m 0.29    \u001b[0m | \u001b[0m 0.2194  \u001b[0m | \u001b[0m 0.2951  \u001b[0m | \u001b[0m 998.3   \u001b[0m |\n",
      "=========================================================================\n"
     ]
    }
   ],
   "source": [
    "SGD_BO.maximize()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'params': {'alpha': 0.002686097902550382,\n",
       "  'epsilon': 0.2776848693533973,\n",
       "  'eta0': 0.08273258247904128,\n",
       "  'max_iter': 388.5705756508965},\n",
       " 'target': 80.82731451083389}"
      ]
     },
     "execution_count": 176,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SGD_BO.max"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Multinomial"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** RandomizedSearchCV **"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = \"MNB\"\n",
    "model = MultinomialNB()\n",
    "parameters = {\n",
    "    'MNB__alpha':[3,2,1, 0.75, 0.8, 0.6, 0.7,0.5, 1e-1, 1e-2,1e-8,10,20,50],\n",
    "    'MNB__fit_prior': [True,False],\n",
    "}\n",
    "_n_iter = 20\n",
    "_cv = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 10 folds for each of 20 candidates, totalling 200 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  33 tasks      | elapsed:    9.5s\n",
      "[Parallel(n_jobs=-1)]: Done 154 tasks      | elapsed:   43.3s\n",
      "[Parallel(n_jobs=-1)]: Done 200 out of 200 | elapsed:   54.2s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "MODEL\n",
      "RandomizedSearchCV(cv=10, error_score=nan,\n",
      "                   estimator=Pipeline(memory=None,\n",
      "                                      steps=[('features',\n",
      "                                              FeatureUnion(n_jobs=None,\n",
      "                                                           transformer_list=[('cant_palabras',\n",
      "                                                                              Pipeline(memory=None,\n",
      "                                                                                       steps=[('selector',\n",
      "                                                                                               NumberSelector(key='#palabras'))],\n",
      "                                                                                       verbose=False)),\n",
      "                                                                             ('links_cant',\n",
      "                                                                              Pipeline(memory=None,\n",
      "                                                                                       steps=[('selector',\n",
      "                                                                                               NumberSelector(key='links_cant'))],\n",
      "                                                                                       verbose=False)),\n",
      "                                                                             ('bo...\n",
      "                                                           verbose=False)),\n",
      "                                             ('MNB',\n",
      "                                              MultinomialNB(alpha=1.0,\n",
      "                                                            class_prior=None,\n",
      "                                                            fit_prior=True))],\n",
      "                                      verbose=False),\n",
      "                   iid='deprecated', n_iter=20, n_jobs=-1,\n",
      "                   param_distributions={'MNB__alpha': [3, 2, 1, 0.75, 0.8, 0.6,\n",
      "                                                       0.7, 0.5, 0.1, 0.01,\n",
      "                                                       1e-08, 10, 20, 50],\n",
      "                                        'MNB__fit_prior': [True, False]},\n",
      "                   pre_dispatch='2*n_jobs', random_state=42, refit=True,\n",
      "                   return_train_score=False, scoring=None, verbose=2)\n",
      "Accuracy = 80.00%.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# RandomizedSearchCV\n",
    "\n",
    "best_params_LR_random = RandomizedSearchCV_models(model_name, model, parameters, X_train.fillna(0), y_train, X_test.fillna(0), y_test, _n_iter, _cv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'MNB__fit_prior': False, 'MNB__alpha': 1}\n"
     ]
    }
   ],
   "source": [
    "print(best_params_LR_random)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** GridSearchCV **"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 10 folds for each of 28 candidates, totalling 280 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  33 tasks      | elapsed:    8.5s\n",
      "[Parallel(n_jobs=-1)]: Done 154 tasks      | elapsed:   38.0s\n",
      "[Parallel(n_jobs=-1)]: Done 280 out of 280 | elapsed:  1.2min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "MODEL\n",
      "GridSearchCV(cv=10, error_score=nan,\n",
      "             estimator=Pipeline(memory=None,\n",
      "                                steps=[('features',\n",
      "                                        FeatureUnion(n_jobs=None,\n",
      "                                                     transformer_list=[('cant_palabras',\n",
      "                                                                        Pipeline(memory=None,\n",
      "                                                                                 steps=[('selector',\n",
      "                                                                                         NumberSelector(key='#palabras'))],\n",
      "                                                                                 verbose=False)),\n",
      "                                                                       ('links_cant',\n",
      "                                                                        Pipeline(memory=None,\n",
      "                                                                                 steps=[('selector',\n",
      "                                                                                         NumberSelector(key='links_cant'))],\n",
      "                                                                                 verbose=False)),\n",
      "                                                                       ('bow',\n",
      "                                                                        Pip...\n",
      "                                                                                 verbose=False))],\n",
      "                                                     transformer_weights=None,\n",
      "                                                     verbose=False)),\n",
      "                                       ('MNB',\n",
      "                                        MultinomialNB(alpha=1.0,\n",
      "                                                      class_prior=None,\n",
      "                                                      fit_prior=True))],\n",
      "                                verbose=False),\n",
      "             iid='deprecated', n_jobs=-1,\n",
      "             param_grid={'MNB__alpha': [3, 2, 1, 0.75, 0.8, 0.6, 0.7, 0.5, 0.1,\n",
      "                                        0.01, 1e-08, 10, 20, 50],\n",
      "                         'MNB__fit_prior': [True, False]},\n",
      "             pre_dispatch='2*n_jobs', refit=True, return_train_score=False,\n",
      "             scoring=None, verbose=2)\n",
      "Accuracy = 80.00%.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# GridSearchCV_models\n",
    "\n",
    "best_params_LR_grid = GridSearchCV_models(model_name, model, parameters, X_train.fillna(0), y_train, X_test.fillna(0), y_test, _cv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'MNB__alpha': 1, 'MNB__fit_prior': False}\n"
     ]
    }
   ],
   "source": [
    "print(best_params_LR_grid)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** Bayesian Optimization **"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [],
   "source": [
    "MNB_parameters = {\n",
    "    'alpha':(1e-2, 5),\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [],
   "source": [
    "def MNB_eval(alpha):\n",
    "  \n",
    "    model = MultinomialNB(\n",
    "        alpha = alpha,\n",
    "        fit_prior = False\n",
    "    )               \n",
    "    \n",
    "    pipe_clf = Pipeline([\n",
    "        ('features', feats),\n",
    "        (\"clf\", model),\n",
    "    ])\n",
    "                                 \n",
    "    pipe_clf.fit(X_train.fillna(0),y_train)\n",
    "                   \n",
    "    return (accuracy_score(y_test, pipe_clf.predict(X_test.fillna(0)))*100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [],
   "source": [
    "MNB_BO = BayesianOptimization(MNB_eval, MNB_parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['alpha']\n"
     ]
    }
   ],
   "source": [
    "print(MNB_BO.space.keys)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "|   iter    |  target   |   alpha   |\n",
      "-------------------------------------\n",
      "| \u001b[0m 1       \u001b[0m | \u001b[0m 78.33   \u001b[0m | \u001b[0m 3.144   \u001b[0m |\n",
      "| \u001b[95m 2       \u001b[0m | \u001b[95m 79.84   \u001b[0m | \u001b[95m 1.795   \u001b[0m |\n",
      "| \u001b[95m 3       \u001b[0m | \u001b[95m 80.24   \u001b[0m | \u001b[95m 1.332   \u001b[0m |\n",
      "| \u001b[0m 4       \u001b[0m | \u001b[0m 77.48   \u001b[0m | \u001b[0m 4.809   \u001b[0m |\n",
      "| \u001b[0m 5       \u001b[0m | \u001b[0m 78.73   \u001b[0m | \u001b[0m 2.903   \u001b[0m |\n",
      "| \u001b[0m 6       \u001b[0m | \u001b[0m 77.87   \u001b[0m | \u001b[0m 0.01    \u001b[0m |\n",
      "| \u001b[0m 7       \u001b[0m | \u001b[0m 80.04   \u001b[0m | \u001b[0m 0.9428  \u001b[0m |\n",
      "| \u001b[0m 8       \u001b[0m | \u001b[0m 80.17   \u001b[0m | \u001b[0m 1.175   \u001b[0m |\n",
      "| \u001b[0m 9       \u001b[0m | \u001b[0m 79.97   \u001b[0m | \u001b[0m 1.494   \u001b[0m |\n",
      "| \u001b[0m 10      \u001b[0m | \u001b[0m 79.32   \u001b[0m | \u001b[0m 2.275   \u001b[0m |\n",
      "| \u001b[0m 11      \u001b[0m | \u001b[0m 77.81   \u001b[0m | \u001b[0m 3.958   \u001b[0m |\n",
      "| \u001b[0m 12      \u001b[0m | \u001b[0m 80.04   \u001b[0m | \u001b[0m 0.9428  \u001b[0m |\n",
      "| \u001b[0m 13      \u001b[0m | \u001b[0m 80.24   \u001b[0m | \u001b[0m 0.6351  \u001b[0m |\n",
      "| \u001b[0m 14      \u001b[0m | \u001b[0m 80.24   \u001b[0m | \u001b[0m 0.7443  \u001b[0m |\n",
      "| \u001b[0m 15      \u001b[0m | \u001b[0m 80.11   \u001b[0m | \u001b[0m 1.269   \u001b[0m |\n",
      "| \u001b[0m 16      \u001b[0m | \u001b[0m 79.51   \u001b[0m | \u001b[0m 2.014   \u001b[0m |\n",
      "| \u001b[0m 17      \u001b[0m | \u001b[0m 79.84   \u001b[0m | \u001b[0m 0.4175  \u001b[0m |\n",
      "| \u001b[0m 18      \u001b[0m | \u001b[0m 78.92   \u001b[0m | \u001b[0m 2.57    \u001b[0m |\n",
      "| \u001b[0m 19      \u001b[0m | \u001b[0m 80.11   \u001b[0m | \u001b[0m 1.069   \u001b[0m |\n",
      "| \u001b[0m 20      \u001b[0m | \u001b[0m 77.94   \u001b[0m | \u001b[0m 3.541   \u001b[0m |\n",
      "| \u001b[0m 21      \u001b[0m | \u001b[0m 80.11   \u001b[0m | \u001b[0m 0.5413  \u001b[0m |\n",
      "| \u001b[0m 22      \u001b[0m | \u001b[0m 80.24   \u001b[0m | \u001b[0m 1.395   \u001b[0m |\n",
      "| \u001b[0m 23      \u001b[0m | \u001b[0m 80.04   \u001b[0m | \u001b[0m 0.6917  \u001b[0m |\n",
      "| \u001b[0m 24      \u001b[0m | \u001b[0m 77.54   \u001b[0m | \u001b[0m 4.369   \u001b[0m |\n",
      "| \u001b[0m 25      \u001b[0m | \u001b[0m 79.71   \u001b[0m | \u001b[0m 1.657   \u001b[0m |\n",
      "| \u001b[95m 26      \u001b[0m | \u001b[95m 80.3    \u001b[0m | \u001b[95m 0.8343  \u001b[0m |\n",
      "| \u001b[0m 27      \u001b[0m | \u001b[0m 79.71   \u001b[0m | \u001b[0m 0.2688  \u001b[0m |\n",
      "| \u001b[95m 28      \u001b[0m | \u001b[95m 80.37   \u001b[0m | \u001b[95m 0.7931  \u001b[0m |\n",
      "| \u001b[0m 29      \u001b[0m | \u001b[0m 80.11   \u001b[0m | \u001b[0m 0.5957  \u001b[0m |\n",
      "| \u001b[0m 30      \u001b[0m | \u001b[0m 77.22   \u001b[0m | \u001b[0m 5.0     \u001b[0m |\n",
      "=====================================\n"
     ]
    }
   ],
   "source": [
    "MNB_BO.maximize()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'params': {'alpha': 0.7931196386847666}, 'target': 80.3676953381484}"
      ]
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "MNB_BO.max"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RandomForestClassifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** RandomizedSearchCV **"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = \"RF\"\n",
    "model = RandomForestClassifier()\n",
    "\n",
    "parameters = {\n",
    "    \"RF__criterion\": [\"gini\", \"entropy\"],\n",
    "    'RF__n_estimators': np.arange(100,500,100),\n",
    "    'RF__max_features': ['auto', \"sqrt\", \"log2\"],\n",
    "    #'RF__max_depth': [np.arange(1, 500, 10)],\n",
    "    #'RF__min_samples_split': np.arange(0, 5, 1),\n",
    "    #'RF__min_samples_leaf': np.arange(0, 5, 1),\n",
    "    #\"RF__min_impurity_decrease\": [0.000001, 0.0001,0.0000001, 0.001, 0.01],\n",
    "    \"RF__n_jobs\": [-1]\n",
    "    #\"RF__min_weight_fraction_leaf\": [0.000001, 0.0001,0.0000001, 0.001, 0.01]\n",
    "}\n",
    "_n_iter = 10\n",
    "_cv = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_params_RF_random = RandomizedSearchCV_models(model_name, model, parameters, X_train.fillna(0), y_train, X_test.fillna(0), y_test, _n_iter, _cv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = \"RF\"\n",
    "model = RandomForestClassifier()\n",
    "\n",
    "parameters = {\n",
    "    \"RF__criterion\": [\"gini\", \"entropy\"],\n",
    "    'RF__n_estimators': np.arange(100,500,100),\n",
    "    'RF__max_features': ['auto', \"sqrt\", \"log2\"],\n",
    "    #'RF__max_depth': [np.arange(1, 500, 10)],\n",
    "    #'RF__min_samples_split': np.arange(0, 5, 1),\n",
    "    #'RF__min_samples_leaf': np.arange(0, 5, 1),\n",
    "    \"RF__ccp_alpha\":[ 0.0001], # Pruning\n",
    "    #\"RF__min_impurity_decrease\": [0.000001, 0.0001,0.0000001, 0.001, 0.01],\n",
    "    \"RF__n_jobs\": [-1]\n",
    "    #\"RF__min_weight_fraction_leaf\": [0.000001, 0.0001,0.0000001, 0.001, 0.01]\n",
    "}\n",
    "_n_iter = 10\n",
    "_cv = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 10 candidates, totalling 30 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m/home/anichu/Programs/anaconda3/lib/python3.6/site-packages/joblib/parallel.py\u001b[0m in \u001b[0;36mretrieve\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    908\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'supports_timeout'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 909\u001b[0;31m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_output\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjob\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    910\u001b[0m                 \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/anichu/Programs/anaconda3/lib/python3.6/site-packages/joblib/_parallel_backends.py\u001b[0m in \u001b[0;36mwrap_future_result\u001b[0;34m(future, timeout)\u001b[0m\n\u001b[1;32m    561\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 562\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfuture\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    563\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mLokyTimeoutError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/anichu/Programs/anaconda3/lib/python3.6/concurrent/futures/_base.py\u001b[0m in \u001b[0;36mresult\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    426\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 427\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_condition\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    428\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/anichu/Programs/anaconda3/lib/python3.6/threading.py\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    294\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 295\u001b[0;31m                 \u001b[0mwaiter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0macquire\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    296\u001b[0m                 \u001b[0mgotit\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: ",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-185-4741e8f1075d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mbest_params_RF_random\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mRandomizedSearchCV_models\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparameters\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX_train\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfillna\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX_test\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfillna\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_n_iter\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_cv\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-48-e6f43f7d3ba5>\u001b[0m in \u001b[0;36mRandomizedSearchCV_models\u001b[0;34m(model_name, model, parameters, X_train, y_train, X_test, y_test, _n_iter, _cv)\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m     \u001b[0mbest_clf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mRandomizedSearchCV\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mestimator\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpipe_clf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparam_distributions\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mparameters\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_iter\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_n_iter\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_cv\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrandom_state\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m42\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_jobs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m     \u001b[0mbest_clf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m     \u001b[0mevaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbest_clf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/anichu/Programs/anaconda3/lib/python3.6/site-packages/sklearn/model_selection/_search.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y, groups, **fit_params)\u001b[0m\n\u001b[1;32m    708\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mresults\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    709\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 710\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_run_search\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mevaluate_candidates\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    711\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    712\u001b[0m         \u001b[0;31m# For multi-metric evaluation, store the best_index_, best_params_ and\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/anichu/Programs/anaconda3/lib/python3.6/site-packages/sklearn/model_selection/_search.py\u001b[0m in \u001b[0;36m_run_search\u001b[0;34m(self, evaluate_candidates)\u001b[0m\n\u001b[1;32m   1482\u001b[0m         evaluate_candidates(ParameterSampler(\n\u001b[1;32m   1483\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparam_distributions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mn_iter\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1484\u001b[0;31m             random_state=self.random_state))\n\u001b[0m",
      "\u001b[0;32m/home/anichu/Programs/anaconda3/lib/python3.6/site-packages/sklearn/model_selection/_search.py\u001b[0m in \u001b[0;36mevaluate_candidates\u001b[0;34m(candidate_params)\u001b[0m\n\u001b[1;32m    687\u001b[0m                                \u001b[0;32mfor\u001b[0m \u001b[0mparameters\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    688\u001b[0m                                in product(candidate_params,\n\u001b[0;32m--> 689\u001b[0;31m                                           cv.split(X, y, groups)))\n\u001b[0m\u001b[1;32m    690\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    691\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/anichu/Programs/anaconda3/lib/python3.6/site-packages/joblib/parallel.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, iterable)\u001b[0m\n\u001b[1;32m   1015\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1016\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mretrieval_context\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1017\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mretrieve\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1018\u001b[0m             \u001b[0;31m# Make sure that we get a last message telling us we are done\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1019\u001b[0m             \u001b[0melapsed_time\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_start_time\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/anichu/Programs/anaconda3/lib/python3.6/site-packages/joblib/parallel.py\u001b[0m in \u001b[0;36mretrieve\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    929\u001b[0m                     \u001b[0;31m# scheduling.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    930\u001b[0m                     \u001b[0mensure_ready\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_managed_backend\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 931\u001b[0;31m                     \u001b[0mbackend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mabort_everything\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mensure_ready\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mensure_ready\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    932\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    933\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexception\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mTransportableException\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/anichu/Programs/anaconda3/lib/python3.6/site-packages/joblib/_parallel_backends.py\u001b[0m in \u001b[0;36mabort_everything\u001b[0;34m(self, ensure_ready)\u001b[0m\n\u001b[1;32m    577\u001b[0m         \"\"\"Shutdown the workers and restart a new one with the same parameters\n\u001b[1;32m    578\u001b[0m         \"\"\"\n\u001b[0;32m--> 579\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_workers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshutdown\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkill_workers\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    580\u001b[0m         \u001b[0mdelete_folder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_workers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_temp_folder\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    581\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_workers\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/anichu/Programs/anaconda3/lib/python3.6/site-packages/joblib/externals/loky/process_executor.py\u001b[0m in \u001b[0;36mshutdown\u001b[0;34m(self, wait, kill_workers)\u001b[0m\n\u001b[1;32m   1099\u001b[0m                     \u001b[0;32mpass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1100\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mwait\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1101\u001b[0;31m                 \u001b[0mqmt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1102\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1103\u001b[0m         \u001b[0mcq\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_queue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/anichu/Programs/anaconda3/lib/python3.6/threading.py\u001b[0m in \u001b[0;36mjoin\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m   1054\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1055\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1056\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_wait_for_tstate_lock\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1057\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1058\u001b[0m             \u001b[0;31m# the behavior of a negative timeout isn't documented, but\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/anichu/Programs/anaconda3/lib/python3.6/threading.py\u001b[0m in \u001b[0;36m_wait_for_tstate_lock\u001b[0;34m(self, block, timeout)\u001b[0m\n\u001b[1;32m   1070\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlock\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# already determined that the C code is done\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1071\u001b[0m             \u001b[0;32massert\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_is_stopped\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1072\u001b[0;31m         \u001b[0;32melif\u001b[0m \u001b[0mlock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0macquire\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mblock\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1073\u001b[0m             \u001b[0mlock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrelease\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1074\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "best_params_RF_random = RandomizedSearchCV_models(model_name, model, parameters, X_train.fillna(0), y_train, X_test.fillna(0), y_test, _n_iter, _cv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(best_params_RF_random)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** GridSearchCV **"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_params_LR_grid = GridSearchCV_models(model_name, model, parameters, X_train.fillna(0), y_train, X_test.fillna(0), y_test, _cv)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** Bayesian Optimization **"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rf_eval(n_estimators, min_samples_split, max_features):\n",
    "    \n",
    "    rfc = RandomForestClassifier(n_estimators=int(n_estimators),\n",
    "                               min_samples_split=int(min_samples_split),\n",
    "                               max_features=min(max_features, 0.999),\n",
    "                               random_state=2,\n",
    "                               n_jobs=-1)\n",
    "    rf= Pipeline([\n",
    "        ('features', feats),\n",
    "        ('randomforest', rfc )\n",
    "        ])\n",
    "    rf.fit(X_train.fillna(0),y_train)\n",
    "    return (accuracy_score(y_test,rf.predict(X_test.fillna(0)))*100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rfBO = BayesianOptimization(rf_eval, {\n",
    "    'n_estimators':(100,500),\n",
    "    'min_samples_split':(2,10),\n",
    "    'max_features':(1,100)\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rfBO.maximize(init_points=1, n_iter=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rfBO.max"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  LGBMClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [],
   "source": [
    "from lightgbm import LGBMClassifier\n",
    "\n",
    "model_name = \"LGBM\"\n",
    "model = LGBMClassifier()\n",
    "\n",
    "parameters ={\n",
    "    'LGBM__objective': ['binary'],\n",
    "    'LGBM__num_leaves': np.arange(25,70,4),\n",
    "    'LGBM__learning_rate':[0.005,0.01,0.05,0.1,0.3],\n",
    "    'LGBM__n_estimators': np.arange(25,200,15),\n",
    "    'LGBM__max_depth': np.arange(5,13,1),\n",
    "    'LGBM__min_split_gain': [0.001,0.01,0.1,0.2],\n",
    "    'LGBM__bagging_fraction': np.arange(0.8,1.01,0.1),\n",
    "    'LGBM__feature_fraction': np.arange(0.1,0.91,0.2)\n",
    "}\n",
    "\n",
    "_n_iter = 10\n",
    "_cv = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 10 folds for each of 10 candidates, totalling 100 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  33 tasks      | elapsed:   12.4s\n",
      "[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   35.8s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "MODEL\n",
      "RandomizedSearchCV(cv=10, error_score=nan,\n",
      "                   estimator=Pipeline(memory=None,\n",
      "                                      steps=[('features',\n",
      "                                              FeatureUnion(n_jobs=None,\n",
      "                                                           transformer_list=[('cant_palabras',\n",
      "                                                                              Pipeline(memory=None,\n",
      "                                                                                       steps=[('selector',\n",
      "                                                                                               NumberSelector(key='#palabras'))],\n",
      "                                                                                       verbose=False)),\n",
      "                                                                             ('links_cant',\n",
      "                                                                              Pipeline(memory=None,\n",
      "                                                                                       steps=[('selector',\n",
      "                                                                                               NumberSelector(key='links_cant'))],\n",
      "                                                                                       verbose=False)),\n",
      "                                                                             ('bo...\n",
      "                                        'LGBM__max_depth': array([ 5,  6,  7,  8,  9, 10, 11, 12]),\n",
      "                                        'LGBM__min_split_gain': [0.001, 0.01,\n",
      "                                                                 0.1, 0.2],\n",
      "                                        'LGBM__n_estimators': array([ 25,  40,  55,  70,  85, 100, 115, 130, 145, 160, 175, 190]),\n",
      "                                        'LGBM__num_leaves': array([25, 29, 33, 37, 41, 45, 49, 53, 57, 61, 65, 69]),\n",
      "                                        'LGBM__objective': ['binary']},\n",
      "                   pre_dispatch='2*n_jobs', random_state=42, refit=True,\n",
      "                   return_train_score=False, scoring=None, verbose=2)\n",
      "Accuracy = 78.00%.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "best_params_LGBM_random = RandomizedSearchCV_models(model_name, model, parameters, X_train.fillna(0), y_train, X_test.fillna(0), y_test, _n_iter, _cv)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### GradientBoostingClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "\n",
    "model_name = \"GBC\"\n",
    "model = GradientBoostingClassifier()\n",
    "\n",
    "parameters = {\n",
    "    'GBC__max_features':['auto', 'sqrt', 'log2'],\n",
    "                             'GBC__max_leaf_nodes': [None,1,2,3,4,5,6,8],\n",
    "                             'GBC__min_weight_fraction_leaf': np.linspace(0.0, 0.5, 1,endpoint=True),\n",
    "                             'GBC__learning_rate': np.arange(0.1, 0.5, 0.05),\n",
    "                             'GBC__min_samples_split': np.linspace(0.1, 1.0, 10, endpoint=True),\n",
    "                             'GBC__min_samples_leaf': np.arange(0.1, 0.5, 5),  \n",
    "                             'GBC__max_features' : list(range(1,100)),\n",
    "                             'GBC__max_depth': [n for n in range(0,50,5)],\n",
    "                             'GBC__n_estimators': [n for n in range(0,1000,10)],\n",
    "                             'GBC__subsample': np.arange(0.3, 1,0.1),\n",
    "                             'GBC__loss': ['deviance'],\n",
    "                             'GBC__warm_start': [True, False],\n",
    "                             'GBC__presort': ['auto'],\n",
    "\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 10 folds for each of 10 candidates, totalling 100 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  33 tasks      | elapsed:   26.5s\n",
      "[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:  1.2min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "MODEL\n",
      "RandomizedSearchCV(cv=10, error_score=nan,\n",
      "                   estimator=Pipeline(memory=None,\n",
      "                                      steps=[('features',\n",
      "                                              FeatureUnion(n_jobs=None,\n",
      "                                                           transformer_list=[('cant_palabras',\n",
      "                                                                              Pipeline(memory=None,\n",
      "                                                                                       steps=[('selector',\n",
      "                                                                                               NumberSelector(key='#palabras'))],\n",
      "                                                                                       verbose=False)),\n",
      "                                                                             ('links_cant',\n",
      "                                                                              Pipeline(memory=None,\n",
      "                                                                                       steps=[('selector',\n",
      "                                                                                               NumberSelector(key='links_cant'))],\n",
      "                                                                                       verbose=False)),\n",
      "                                                                             ('bo...\n",
      "                                        'GBC__min_weight_fraction_leaf': array([0.]),\n",
      "                                        'GBC__n_estimators': [0, 10, 20, 30, 40,\n",
      "                                                              50, 60, 70, 80,\n",
      "                                                              90, 100, 110, 120,\n",
      "                                                              130, 140, 150,\n",
      "                                                              160, 170, 180,\n",
      "                                                              190, 200, 210,\n",
      "                                                              220, 230, 240,\n",
      "                                                              250, 260, 270,\n",
      "                                                              280, 290, ...],\n",
      "                                        'GBC__presort': ['auto'],\n",
      "                                        'GBC__subsample': array([0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]),\n",
      "                                        'GBC__warm_start': [True, False]},\n",
      "                   pre_dispatch='2*n_jobs', random_state=42, refit=True,\n",
      "                   return_train_score=False, scoring=None, verbose=2)\n",
      "Accuracy = 56.00%.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "best_params_GBC_random = RandomizedSearchCV_models(model_name, model, parameters, X_train.fillna(0), y_train, X_test.fillna(0), y_test, _n_iter, _cv)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### DesicionTreeClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "model_name = \"DT\"\n",
    "model = DecisionTreeClassifier()\n",
    "\n",
    "parameters = {\n",
    "    \"DT__criterion\": [\"entropy\",\"gini\"],\n",
    "    'DT__max_features': ['auto', 'sqrt','log2'],\n",
    "    'DT__max_depth': np.linspace(1, 500, 10, endpoint=True),\n",
    "    'DT__min_samples_split': np.linspace(0.0, 1.0, 10, endpoint=True),\n",
    "    'DT__min_samples_leaf': np.arange(0.0, 1,0.05),\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 10 folds for each of 10 candidates, totalling 100 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  33 tasks      | elapsed:    5.9s\n",
      "[Parallel(n_jobs=-1)]: Done 100 out of 100 | elapsed:   17.0s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "MODEL\n",
      "RandomizedSearchCV(cv=10, error_score=nan,\n",
      "                   estimator=Pipeline(memory=None,\n",
      "                                      steps=[('features',\n",
      "                                              FeatureUnion(n_jobs=None,\n",
      "                                                           transformer_list=[('cant_palabras',\n",
      "                                                                              Pipeline(memory=None,\n",
      "                                                                                       steps=[('selector',\n",
      "                                                                                               NumberSelector(key='#palabras'))],\n",
      "                                                                                       verbose=False)),\n",
      "                                                                             ('links_cant',\n",
      "                                                                              Pipeline(memory=None,\n",
      "                                                                                       steps=[('selector',\n",
      "                                                                                               NumberSelector(key='links_cant'))],\n",
      "                                                                                       verbose=False)),\n",
      "                                                                             ('bo...\n",
      "                                                             'log2'],\n",
      "                                        'DT__min_samples_leaf': array([0.  , 0.05, 0.1 , 0.15, 0.2 , 0.25, 0.3 , 0.35, 0.4 , 0.45, 0.5 ,\n",
      "       0.55, 0.6 , 0.65, 0.7 , 0.75, 0.8 , 0.85, 0.9 , 0.95]),\n",
      "                                        'DT__min_samples_split': array([0.        , 0.11111111, 0.22222222, 0.33333333, 0.44444444,\n",
      "       0.55555556, 0.66666667, 0.77777778, 0.88888889, 1.        ])},\n",
      "                   pre_dispatch='2*n_jobs', random_state=42, refit=True,\n",
      "                   return_train_score=False, scoring=None, verbose=2)\n",
      "Accuracy = 56.00%.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "best_params_GBC_random = RandomizedSearchCV_models(model_name, model, parameters, X_train.fillna(0), y_train, X_test.fillna(0), y_test, _n_iter, _cv)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### XGBClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = \"XGB\"\n",
    "model = XGBClassifier()\n",
    "\n",
    "parameters ={\n",
    "    \"XGB__criterion\": [\"entropy\",\"gini\"],\n",
    "    'XGB__n_estimators': np.arange(10,2000,200),\n",
    "    'XGB__max_features': ['auto', 'sqrt','log2'],\n",
    "    'XGB__max_depth': np.arange(1, 500, 10),\n",
    "    'XGB__min_samples_split': np.linspace(0.0, 1.0, 10, endpoint=True),\n",
    "    'XGB__min_samples_leaf': np.arange(1, 10, 1),\n",
    "    'XGB__objective': ['binary:logistic'],\n",
    "    'XGB__learning_rate':np.arange(0.1,0.5,0.1),\n",
    "    'XGB__gamma':np.arange(0,0.5,0.1),\n",
    "    'XGB__subsample':np.arange(0.6,1,0.1),\n",
    "    'XGB__reg_alpha':[1e-5, 1e-2, 0.1, 1, 100],\n",
    "    'XGB__tree_method':['auto', 'exact', 'approx', 'hist', 'gpu_hist']\n",
    "}\n",
    "\n",
    "_n_iter = 11\n",
    "_cv = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "best_params_XGB_random = RandomizedSearchCV_models(model_name, model, parameters, X_train.fillna(0), y_train, X_test.fillna(0), y_test, _n_iter, _cv)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### MLPClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = \"MLPC\"\n",
    "model = MLPClassifier()\n",
    "\n",
    "parameters = {'MLPC__hidden_layer_sizes':[(n,n,n) for n in range(1,30)],\n",
    "                'MLPC__activation':['identity', 'logistic', 'tanh', 'relu'],\n",
    "                              'MLPC__alpha':[1e-06,1e-05,1e-04,1e-03,1e-02,1e-01,1],\n",
    "                              'MLPC__beta_1':np.arange(0.0,1,0.01),\n",
    "                              'MLPC__beta_2':np.arange(0.0,1,0.01),\n",
    "                              'MLPC__early_stopping':[True, False],\n",
    "                              'MLPC__epsilon':[1e-07,1e-08,1e-06, 1e-09, 1e-010, 1e-11],\n",
    "                              'MLPC__learning_rate':['constant', 'adaptive'],\n",
    "                              'MLPC__solver':['adam', 'lbfgs', 'sgd'],\n",
    "                              'MLPC__validation_fraction':np.arange(0.15,0.5,0.01),\n",
    "                               'MLPC__max_iter':[200,300,400],\n",
    "                             }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_params_MLPC_random = RandomizedSearchCV_models(model_name, model, parameters, X_train.fillna(0), y_train, X_test.fillna(0), y_test, _n_iter, _cv)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PREDICT ON TEST AND SAVE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# replace \"model\" with the final model name\n",
    "model.fit(X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# replace \"model\" with the final model name and name the file (dont forget the timestamp!)\n",
    "save_prediction(model, test.text, 'some_name.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Vectorizacion del texto con Word2vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7613, 300)"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gensim\n",
    "url = 'GoogleNews-vectors-negative300.bin.gz'\n",
    "embeddings = gensim.models.KeyedVectors.load_word2vec_format(url,binary=True)\n",
    "data = pd.DataFrame()\n",
    "for frase in train.text:\n",
    "    temp = pd.DataFrame()\n",
    "    for word in frase.split(' '):\n",
    "        try:\n",
    "            word_vec = embeddings[word]\n",
    "            temp = temp.append(pd.Series(word_vec), ignore_index=True)\n",
    "        except:\n",
    "            pass\n",
    "    mean = temp.mean()\n",
    "    data = data.append(mean, ignore_index=True)\n",
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5700"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.isnull(data).sum().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['target'] = train['target']\n",
    "data = data.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((6075, 300), (1519, 300), (6075,), (1519,))"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_X, test_X, train_y, test_y = train_test_split(data.drop('target',axis=1), data['target'], test_size=0.2, random_state=1)\n",
    "train_X.shape, test_X.shape, train_y.shape, test_y.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Modelos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neural_network import MLPClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mlp = MLPClassifier()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_prediction_accuracy "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-9fce0673cc39>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mensemble\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mAdaBoostClassifier\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mAdaBoostClassifier\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn_estimators\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m800\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrandom_state\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_X\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mtrain_y\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/sklearn/ensemble/__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0m_base\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mBaseEnsemble\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0m_forest\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mRandomForestClassifier\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      9\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0m_forest\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mRandomForestRegressor\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0m_forest\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mRandomTreesEmbedding\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/sklearn/ensemble/_forest.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     54\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmetrics\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mr2_score\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     55\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpreprocessing\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mOneHotEncoder\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 56\u001b[0;31m from ..tree import (DecisionTreeClassifier, DecisionTreeRegressor,\n\u001b[0m\u001b[1;32m     57\u001b[0m                     ExtraTreeClassifier, ExtraTreeRegressor)\n\u001b[1;32m     58\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtree\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_tree\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mDTYPE\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mDOUBLE\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/sklearn/tree/__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      4\u001b[0m \"\"\"\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0m_classes\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mBaseDecisionTree\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0m_classes\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mDecisionTreeClassifier\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0m_classes\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mDecisionTreeRegressor\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/sklearn/tree/_classes.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     39\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mutils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalidation\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0m_deprecate_positional_args\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     40\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 41\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0m_criterion\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mCriterion\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     42\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0m_splitter\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mSplitter\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     43\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0m_tree\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mDepthFirstTreeBuilder\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32msklearn/tree/_criterion.pyx\u001b[0m in \u001b[0;36minit sklearn.tree._criterion\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32msklearn/tree/_splitter.pyx\u001b[0m in \u001b[0;36minit sklearn.tree._splitter\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32msklearn/tree/_tree.pyx\u001b[0m in \u001b[0;36minit sklearn.tree._tree\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/sklearn/neighbors/__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     15\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0m_kde\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mKernelDensity\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0m_lof\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mLocalOutlierFactor\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 17\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0m_nca\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mNeighborhoodComponentsAnalysis\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     18\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0m_base\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mVALID_METRICS\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mVALID_METRICS_SPARSE\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/sklearn/neighbors/_nca.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     20\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbase\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mBaseEstimator\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mTransformerMixin\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     21\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpreprocessing\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mLabelEncoder\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 22\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdecomposition\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mPCA\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     23\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mutils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmulticlass\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mcheck_classification_targets\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mutils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrandom\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mcheck_random_state\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/sklearn/decomposition/__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     15\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mwarnings\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcatch_warnings\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m     \u001b[0mwarnings\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msimplefilter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"ignore\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcategory\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mFutureWarning\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 17\u001b[0;31m     \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mdict_learning\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mdict_learning\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     18\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/sklearn/decomposition/dict_learning.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;31m# mypy error: Module X has no attribute y (typically for C extensions)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0m_dict_learning\u001b[0m  \u001b[0;31m# type: ignore\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexternals\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_pep562\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mPep562\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mutils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdeprecation\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0m_raise_dep_warning_if_not_pytest\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/importlib/_bootstrap.py\u001b[0m in \u001b[0;36m_find_and_load\u001b[0;34m(name, import_)\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/importlib/_bootstrap.py\u001b[0m in \u001b[0;36m_find_and_load_unlocked\u001b[0;34m(name, import_)\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/importlib/_bootstrap.py\u001b[0m in \u001b[0;36m_load_unlocked\u001b[0;34m(spec)\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/importlib/_bootstrap_external.py\u001b[0m in \u001b[0;36mexec_module\u001b[0;34m(self, module)\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/importlib/_bootstrap_external.py\u001b[0m in \u001b[0;36mget_code\u001b[0;34m(self, fullname)\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/importlib/_bootstrap_external.py\u001b[0m in \u001b[0;36mget_data\u001b[0;34m(self, path)\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "\n",
    "model=AdaBoostClassifier(n_estimators=800, random_state=1)\n",
    "model.fit(train_X,train_y)\n",
    "\n",
    "predictions = model.predict(test_X)\n",
    "accuracy = round(accuracy_score(test_y,predictions)*100)\n",
    "print('Model Performance')\n",
    "print('Accuracy = {:0.2f}%.'.format(accuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model Performance\n",
      "Accuracy = 78.00%.\n"
     ]
    }
   ],
   "source": [
    "model = LogisticRegression()\n",
    "model.fit(train_X,train_y)\n",
    "\n",
    "predictions = model.predict(test_X)\n",
    "accuracy = round(accuracy_score(test_y,predictions)*100)\n",
    "print('Model Performance')\n",
    "print('Accuracy = {:0.2f}%.'.format(accuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model Performance\n",
      "Accuracy = 78.00%.\n"
     ]
    }
   ],
   "source": [
    "model = RandomForestClassifier()\n",
    "model.fit(train_X,train_y)\n",
    "\n",
    "predictions = model.predict(test_X)\n",
    "accuracy = round(accuracy_score(test_y,predictions)*100)\n",
    "print('Model Performance')\n",
    "print('Accuracy = {:0.2f}%.'.format(accuracy))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}