CP2_06_RandomForestModelling.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"CP2_06_RandomForestModelling.ipynb","provenance":[{"file_id":"11Wcj6NclP2KX26XYfFgEudr0S0bFEp9A","timestamp":1617497432616},{"file_id":"1t9YTFMUahjeBWsyCculou_Pynkh2InG6","timestamp":1602256804263},{"file_id":"12PjhOUiDi-N0FXtlN7AFXyyGAiYi7AWj","timestamp":1601078329171},{"file_id":"1U1u9m5zkvqaI5GElpVlY4tU7zTNShmq9","timestamp":1599859230341},{"file_id":"1JbxiYiHD6fO2SZ9yLDoyqe9IkvHwJxp8","timestamp":1599777923777}],"collapsed_sections":["hMm7TNpqTCwM","JKLAl0zlukPE","-iPLi5m886bu","fCmKiWgw9GiF","qkAIQnqGU1qd","2xDBbNOou7zF","MzSC5VNF2p89"],"machine_shape":"hm"},"kernelspec":{"name":"python3","display_name":"Python 3"}},"cells":[{"cell_type":"code","metadata":{"id":"vldA80BR2gik","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1621714645726,"user_tz":360,"elapsed":20842,"user":{"displayName":"Laura Elliott","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiR_lgn2QjRHbMFy-LVQ-MZiwLY0GVJVCQPQcKj=s64","userId":"17671003486597139296"}},"outputId":"ea20994a-e69c-4e80-a17d-ec96b41d4af8"},"source":["from google.colab import drive \n","drive.mount('/content/drive')"],"execution_count":1,"outputs":[{"output_type":"stream","text":["Mounted at /content/drive\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"BMCdDFsXr5rR"},"source":["# PseudoCode and Task List"]},{"cell_type":"markdown","metadata":{"id":"kxaQxoftDTp3"},"source":["1. Load the pickled pandas dataframe from 04 notebook and check file contents\n","2. Prep the dataset for analysis\n",">2a. Factorize Tags column to a numeric column\n",">2b. Split into dev, cv, and test sets\n","3. First of 3 different models (Random Forest) run each with the 3 diff vectorizors - evaluate with AUC scores on the val set of each transformation -\n",">3.1 Count Vectorization\n",">3.2 TFIDF\n",">3.3 Doc2Vec \n","4. TFIDF gave slightly higher auc scores on the cv set - all models are highly overfitted however; a final hyperparam tuning step performed to reduce overfitting was attempted\n","\n"]},{"cell_type":"markdown","metadata":{"id":"gJ3VoPrnsIkz"},"source":["# Task 1 Load file and examine contents"]},{"cell_type":"markdown","metadata":{"id":"ganmqxEOH4p2"},"source":[""]},{"cell_type":"code","metadata":{"id":"vt8U0qUv28dY","executionInfo":{"status":"ok","timestamp":1621714652758,"user_tz":360,"elapsed":3049,"user":{"displayName":"Laura Elliott","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiR_lgn2QjRHbMFy-LVQ-MZiwLY0GVJVCQPQcKj=s64","userId":"17671003486597139296"}}},"source":["'''\n","1a Import all modules that are needed\n","'''\n","import pandas as pd\n","import pickle\n","import numpy as np\n","import matplotlib.pyplot as plt\n","import seaborn as sns\n","import collections\n","import re\n","import nltk\n","import itertools\n","from sklearn.feature_extraction.text import CountVectorizer\n","from sklearn.feature_extraction.text import TfidfVectorizer\n","from sklearn.ensemble import RandomForestClassifier\n","from sklearn.model_selection import cross_val_score\n","from sklearn.model_selection import train_test_split\n","from sklearn.model_selection import LeaveOneOut \n","from sklearn import metrics\n","from sklearn.metrics import roc_auc_score\n","from sklearn.metrics import classification_report\n","from sklearn.metrics import confusion_matrix\n","from sklearn.model_selection import GridSearchCV\n","from sklearn.pipeline import Pipeline\n","from gensim.models.doc2vec import Doc2Vec, TaggedDocument\n","from gensim.parsing.preprocessing import preprocess_documents\n","from gensim.parsing.preprocessing import preprocess_string\n","from prettytable import PrettyTable"],"execution_count":2,"outputs":[]},{"cell_type":"code","metadata":{"id":"LIOI0tDD3Ke7","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1621714657335,"user_tz":360,"elapsed":2192,"user":{"displayName":"Laura Elliott","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiR_lgn2QjRHbMFy-LVQ-MZiwLY0GVJVCQPQcKj=s64","userId":"17671003486597139296"}},"outputId":"726efd09-4ef2-415c-e916-c00748b83b84"},"source":["'''\n","1b Load file \n","<class 'pandas.core.frame.DataFrame'>\n","Int64Index: 24330 entries, 0 to 24352\n","Data columns (total 3 columns):\n"," #   Column          Non-Null Count  Dtype \n","---  ------          --------------  ----- \n"," 0   Id              24330 non-null  int64 \n"," 1   Tag             24330 non-null  object\n"," 2   BodyText_Clean  24330 non-null  object\n","dtypes: int64(1), object(2)\n","memory usage: 760.3+ KB\n","'''\n","\n","questions_df_clean = pd.read_pickle('/content/drive/My Drive/Capstone2/Data/questions_df_clean_11052020.pickle')\n","#questions_df_clean = pd.read_pickle('/content/drive/MyDrive/Data Science/Laura_CP2/Copy of questions_df_clean_11052020.pickle')\n","questions_df_clean.info()"],"execution_count":3,"outputs":[{"output_type":"stream","text":["<class 'pandas.core.frame.DataFrame'>\n","Int64Index: 24330 entries, 0 to 24352\n","Data columns (total 3 columns):\n"," #   Column          Non-Null Count  Dtype \n","---  ------          --------------  ----- \n"," 0   Id              24330 non-null  int64 \n"," 1   Tag             24330 non-null  object\n"," 2   BodyText_Clean  24330 non-null  object\n","dtypes: int64(1), object(2)\n","memory usage: 760.3+ KB\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"rYJBGuegELBW","colab":{"base_uri":"https://localhost:8080/","height":198},"executionInfo":{"status":"ok","timestamp":1621714660706,"user_tz":360,"elapsed":177,"user":{"displayName":"Laura Elliott","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiR_lgn2QjRHbMFy-LVQ-MZiwLY0GVJVCQPQcKj=s64","userId":"17671003486597139296"}},"outputId":"5676077f-f66e-468c-e95c-71e10b1c5a71"},"source":["'''\n","1c. Examine contents\n","'''\n","questions_df_clean.head()\n","#questions_df_clean.loc[questions_df_clean['BodyText_Clean'].isnull()]\n","#questions_df_clean.loc[questions_df_clean['Tag'].isnull()]"],"execution_count":4,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>Id</th>\n","      <th>Tag</th>\n","      <th>BodyText_Clean</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>5</td>\n","      <td>machine-learning</td>\n","      <td>always interest machine learn figure one thing...</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>7</td>\n","      <td>Other</td>\n","      <td>researcher instructor look opensource book sim...</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>14</td>\n","      <td>data-mining</td>\n","      <td>sure data science discus forum several synonym...</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>15</td>\n","      <td>Other</td>\n","      <td>situation would one system prefer relative adv...</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>16</td>\n","      <td>machine-learning</td>\n","      <td>use libsvm train data predict classification s...</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["   Id               Tag                                     BodyText_Clean\n","0   5  machine-learning  always interest machine learn figure one thing...\n","1   7             Other  researcher instructor look opensource book sim...\n","2  14       data-mining  sure data science discus forum several synonym...\n","3  15             Other  situation would one system prefer relative adv...\n","4  16  machine-learning  use libsvm train data predict classification s..."]},"metadata":{"tags":[]},"execution_count":4}]},{"cell_type":"markdown","metadata":{"id":"Uxa_Yv16OpYL"},"source":["# Task 2 - Prep for modelling\n",">2a. Factorize Tags column to numeric column\n",">2b. Split into dev, cv, and test sets\n",">2c. Verify the distribution of tags within the splits"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"YEig4gYOOx-n","executionInfo":{"status":"ok","timestamp":1621714665037,"user_tz":360,"elapsed":197,"user":{"displayName":"Laura Elliott","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiR_lgn2QjRHbMFy-LVQ-MZiwLY0GVJVCQPQcKj=s64","userId":"17671003486597139296"}},"outputId":"70eba934-ef37-4cd1-b1f5-04118a0dbf7f"},"source":["''' \n","2a. Factorize Tags Column to numeric\n","Converting tag column (our target variable) to a numeric column for modelling; \n","originally converted to separate columns for each tag\n","that process returned 1(yes) or 0(no) for each tag name in the original Tag column\n","multiple y target variables are represented by these multi-labelled columns\n","returning a list of all these multi-label target columns\n","That initial process did not work well using the below code\n","tag_names = questions_df_clean['Tag'].unique().tolist()\n","#print(tag_names)\n","tag_dummy = pd.get_dummies(questions_df_clean['Tag'], prefix = 'Tag')\n","quest_df_dummies = pd.concat([questions_df_clean, tag_dummy], axis = 1)\n","quest_df_dummies.drop(columns='Tag', inplace=True)\n","quest_df_dummies.info()\n","Using factorize code instead suggested by Ajith\n","'''\n","questions_df_factorized = questions_df_clean.copy()\n","#Creating the dependent variable class\n","factor = pd.factorize(questions_df_factorized['Tag'])\n","questions_df_factorized.Tag = factor[0]\n","definitions = factor[1]\n","print(questions_df_factorized.head())\n","print(questions_df_clean.head())\n","print(definitions)\n","print(factor)"],"execution_count":5,"outputs":[{"output_type":"stream","text":["   Id  Tag                                     BodyText_Clean\n","0   5    0  always interest machine learn figure one thing...\n","1   7    1  researcher instructor look opensource book sim...\n","2  14    2  sure data science discus forum several synonym...\n","3  15    1  situation would one system prefer relative adv...\n","4  16    0  use libsvm train data predict classification s...\n","   Id               Tag                                     BodyText_Clean\n","0   5  machine-learning  always interest machine learn figure one thing...\n","1   7             Other  researcher instructor look opensource book sim...\n","2  14       data-mining  sure data science discus forum several synonym...\n","3  15             Other  situation would one system prefer relative adv...\n","4  16  machine-learning  use libsvm train data predict classification s...\n","Index(['machine-learning', 'Other', 'data-mining', 'bigdata', 'r',\n","       'statistics', 'clustering', 'recommender-system', 'nlp',\n","       'feature-selection', 'neural-network', 'python', 'classification',\n","       'visualization', 'apache-spark', 'dataset', 'text-mining', 'pandas',\n","       'time-series', 'regression', 'random-forest', 'predictive-modeling',\n","       'scikit-learn', 'deep-learning', 'decision-trees', 'linear-regression',\n","       'data', 'xgboost', 'tensorflow', 'reinforcement-learning', 'orange',\n","       'keras', 'cnn', 'lstm'],\n","      dtype='object')\n","(array([ 0,  1,  2, ...,  0, 29, 11]), Index(['machine-learning', 'Other', 'data-mining', 'bigdata', 'r',\n","       'statistics', 'clustering', 'recommender-system', 'nlp',\n","       'feature-selection', 'neural-network', 'python', 'classification',\n","       'visualization', 'apache-spark', 'dataset', 'text-mining', 'pandas',\n","       'time-series', 'regression', 'random-forest', 'predictive-modeling',\n","       'scikit-learn', 'deep-learning', 'decision-trees', 'linear-regression',\n","       'data', 'xgboost', 'tensorflow', 'reinforcement-learning', 'orange',\n","       'keras', 'cnn', 'lstm'],\n","      dtype='object'))\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"YVJfJnIRcQKz","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1621714670033,"user_tz":360,"elapsed":169,"user":{"displayName":"Laura Elliott","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiR_lgn2QjRHbMFy-LVQ-MZiwLY0GVJVCQPQcKj=s64","userId":"17671003486597139296"}},"outputId":"1b3d3ed4-8ad1-46b2-a32b-71330ee86340"},"source":["'''\n","2b. Split into train (70%) / test (30%). Use the train data and further split into train/val split (similar ratio). \n","Leaving the test split to the end.\n","Splitting into target (y) and predictor (X) variable sets and then into \n","test and train sets and using stratification, given that the tag distribution is imbalanced\n","Experimented with various means to deal with stratification and multi-label classification\n","and decided the standard scikit learn module code works better\n","\n","from sklearn.preprocessing import MultiLabelBinarizer\n","mlb = MultiLabelBinarizer()\n","X=df[list('ABCD')]\n","Y=pd.DataFrame(mlb.fit_transform(df[['sex','weight']].values), columns=mlb.classes_, index=df.index)\n","\n","!pip install scikit-multilearn\n","from skmultilearn.model_selection import iterative_stratification\n","X_train, y_train, X_test, y_test = iterative_train_test_split(X, y, test_size = 0.30)\n","\n","'''\n","# Splitting X and y variables\n","X=questions_df_factorized[list(questions_df_factorized.columns)[2]]\n","y=questions_df_factorized[list(questions_df_factorized.columns)[1]]\n","print(X.shape)\n","print(y.shape)\n","# Splitting into train, test\n","X_train, X_test, y_train, y_test = train_test_split(X,y,test_size = 0.30, stratify=y,random_state=42)\n","print(X_train.shape, y_train.shape)\n","print(X_test.shape, y_test.shape)\n","# Further splitting train into dev and validation\n","X_dev, X_cv, y_dev, y_cv = train_test_split(X_train,y_train,test_size = 0.30,stratify=y_train,random_state=42)\n","print(X_dev.shape, y_dev.shape)\n","print(X_cv.shape, y_cv.shape)\n"],"execution_count":6,"outputs":[{"output_type":"stream","text":["(24330,)\n","(24330,)\n","(17031,) (17031,)\n","(7299,) (7299,)\n","(11921,) (11921,)\n","(5110,) (5110,)\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"_7T0Gmd0yBDT","executionInfo":{"status":"ok","timestamp":1621714673224,"user_tz":360,"elapsed":204,"user":{"displayName":"Laura Elliott","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiR_lgn2QjRHbMFy-LVQ-MZiwLY0GVJVCQPQcKj=s64","userId":"17671003486597139296"}},"outputId":"44ea3b02-86f1-4c7a-c44c-dffca05fbf59"},"source":["print(type(y_dev))\n","y_dev.isnull().sum()"],"execution_count":7,"outputs":[{"output_type":"stream","text":["<class 'pandas.core.series.Series'>\n"],"name":"stdout"},{"output_type":"execute_result","data":{"text/plain":["0"]},"metadata":{"tags":[]},"execution_count":7}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":198},"id":"NKZDPGl7DdLb","executionInfo":{"status":"ok","timestamp":1621714675002,"user_tz":360,"elapsed":5,"user":{"displayName":"Laura Elliott","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiR_lgn2QjRHbMFy-LVQ-MZiwLY0GVJVCQPQcKj=s64","userId":"17671003486597139296"}},"outputId":"a47e2047-d145-47ea-e80c-f540a1f6d1bf"},"source":["X_dev = pd.DataFrame(X_dev)\n","X_cv = pd.DataFrame(X_cv)\n","X_test = pd.DataFrame(X_test)\n","X_train = pd.DataFrame(X_train)\n","X_dev.head()"],"execution_count":8,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>BodyText_Clean</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>8286</th>\n","      <td>seems like thing httpswwwsciencedirectcomscien...</td>\n","    </tr>\n","    <tr>\n","      <th>23415</th>\n","      <td>build 2hidden layer mlp use keras use scikit l...</td>\n","    </tr>\n","    <tr>\n","      <th>23360</th>\n","      <td>write fast rcnn run problem back propagation g...</td>\n","    </tr>\n","    <tr>\n","      <th>1698</th>\n","      <td>hear multilayer perceptron approximate functio...</td>\n","    </tr>\n","    <tr>\n","      <th>24085</th>\n","      <td>try correlation analysis dataset data cleanse ...</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["                                          BodyText_Clean\n","8286   seems like thing httpswwwsciencedirectcomscien...\n","23415  build 2hidden layer mlp use keras use scikit l...\n","23360  write fast rcnn run problem back propagation g...\n","1698   hear multilayer perceptron approximate functio...\n","24085  try correlation analysis dataset data cleanse ..."]},"metadata":{"tags":[]},"execution_count":8}]},{"cell_type":"code","metadata":{"id":"AvYWS9eBDQpV","executionInfo":{"status":"ok","timestamp":1621714679535,"user_tz":360,"elapsed":1024,"user":{"displayName":"Laura Elliott","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiR_lgn2QjRHbMFy-LVQ-MZiwLY0GVJVCQPQcKj=s64","userId":"17671003486597139296"}}},"source":["X_dev['BodyText_Clean'] = X_dev['BodyText_Clean'].apply(lambda x: re.sub(\"^\\d+\\s|\\s\\d+\\s|\\s\\d+$\", \" \", x))\n","X_cv['BodyText_Clean'] = X_cv['BodyText_Clean'].apply(lambda x: re.sub(\"^\\d+\\s|\\s\\d+\\s|\\s\\d+$\", \" \", x))\n","X_test['BodyText_Clean'] = X_test['BodyText_Clean'].apply(lambda x: re.sub(\"^\\d+\\s|\\s\\d+\\s|\\s\\d+$\", \" \", x))\n","X_train['BodyText_Clean'] = X_train['BodyText_Clean'].apply(lambda x: re.sub(\"^\\d+\\s|\\s\\d+\\s|\\s\\d+$\", \" \", x))"],"execution_count":9,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"-iPLi5m886bu"},"source":["# Task 3 - Build first of 3 different models (Random Forest) and run with the 3 diff transformations -  measure the accuracy on the val set for each \n"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":264},"id":"vzQwLpY0sZ5l","executionInfo":{"status":"ok","timestamp":1621714685181,"user_tz":360,"elapsed":1250,"user":{"displayName":"Laura Elliott","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiR_lgn2QjRHbMFy-LVQ-MZiwLY0GVJVCQPQcKj=s64","userId":"17671003486597139296"}},"outputId":"a06387ff-bae8-4320-c95f-6e4bf8140ab0"},"source":["'''\n","3.1a. Running count vectorizer on dev and cv sets with optimal params\n","'''\n","cnt_vect = CountVectorizer(min_df=.005,max_df=.99, ngram_range=(1, 1))\n","X_dev_cntvect_df = pd.DataFrame(cnt_vect.fit_transform(X_dev.BodyText_Clean).toarray(), index=X_dev.index, columns=cnt_vect.get_feature_names())\n","print(X_dev_cntvect_df.shape)\n","X_dev_cntvect_df.head()\n","X_cv_cntvect = cnt_vect.transform(X_cv.BodyText_Clean)\n","\n","# Convert cv sets using the same transformation\n","\n","X_cv_cntvect_df = pd.DataFrame(cnt_vect.transform(X_cv.BodyText_Clean).toarray(), index = X_cv.index, columns = cnt_vect.get_feature_names())\n","X_cv_cntvect_df.head()"],"execution_count":10,"outputs":[{"output_type":"stream","text":["(11921, 1231)\n"],"name":"stdout"},{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>1d</th>\n","      <th>2d</th>\n","      <th>2nd</th>\n","      <th>3d</th>\n","      <th>able</th>\n","      <th>absolute</th>\n","      <th>accept</th>\n","      <th>access</th>\n","      <th>accomplish</th>\n","      <th>accord</th>\n","      <th>according</th>\n","      <th>account</th>\n","      <th>accuracy</th>\n","      <th>accurate</th>\n","      <th>achieve</th>\n","      <th>across</th>\n","      <th>action</th>\n","      <th>activation</th>\n","      <th>activity</th>\n","      <th>actual</th>\n","      <th>actually</th>\n","      <th>adam</th>\n","      <th>add</th>\n","      <th>addition</th>\n","      <th>additional</th>\n","      <th>address</th>\n","      <th>adjust</th>\n","      <th>advance</th>\n","      <th>advantage</th>\n","      <th>advice</th>\n","      <th>advise</th>\n","      <th>affect</th>\n","      <th>age</th>\n","      <th>agent</th>\n","      <th>aggregate</th>\n","      <th>ai</th>\n","      <th>aim</th>\n","      <th>al</th>\n","      <th>algorithm</th>\n","      <th>allow</th>\n","      <th>...</th>\n","      <th>visualization</th>\n","      <th>visualize</th>\n","      <th>want</th>\n","      <th>way</th>\n","      <th>web</th>\n","      <th>website</th>\n","      <th>week</th>\n","      <th>weight</th>\n","      <th>weird</th>\n","      <th>welcome</th>\n","      <th>well</th>\n","      <th>whereas</th>\n","      <th>whether</th>\n","      <th>whole</th>\n","      <th>whose</th>\n","      <th>width</th>\n","      <th>wikipedia</th>\n","      <th>win</th>\n","      <th>window</th>\n","      <th>wish</th>\n","      <th>within</th>\n","      <th>without</th>\n","      <th>wonder</th>\n","      <th>word</th>\n","      <th>word2vec</th>\n","      <th>work</th>\n","      <th>world</th>\n","      <th>worth</th>\n","      <th>would</th>\n","      <th>write</th>\n","      <th>wrong</th>\n","      <th>x1</th>\n","      <th>x2</th>\n","      <th>xgboost</th>\n","      <th>xi</th>\n","      <th>year</th>\n","      <th>yes</th>\n","      <th>yet</th>\n","      <th>yield</th>\n","      <th>zero</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>20034</th>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>22608</th>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>12975</th>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>2</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>614</th>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>2</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>2</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>3</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>22615</th>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>...</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>5 rows × 1231 columns</p>\n","</div>"],"text/plain":["       1d  2d  2nd  3d  able  absolute  ...  xi  year  yes  yet  yield  zero\n","20034   0   0    0   0     0         0  ...   0     0    0    0      0     0\n","22608   0   0    0   0     0         0  ...   0     0    0    0      0     0\n","12975   0   0    0   0     2         0  ...   0     0    0    0      0     0\n","614     0   0    0   0     0         0  ...   0     0    0    0      0     0\n","22615   0   0    0   0     0         0  ...   0     0    0    0      0     0\n","\n","[5 rows x 1231 columns]"]},"metadata":{"tags":[]},"execution_count":10}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"X9_3pYd-vgD3","executionInfo":{"status":"ok","timestamp":1621714689556,"user_tz":360,"elapsed":1474,"user":{"displayName":"Laura Elliott","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiR_lgn2QjRHbMFy-LVQ-MZiwLY0GVJVCQPQcKj=s64","userId":"17671003486597139296"}},"outputId":"1015e9c4-a6ea-4a74-8d81-d24ec6fe4341"},"source":["'''\n","3.1b. Running base random forest model\n","'''\n","rf_base_cv  = RandomForestClassifier(n_estimators=10, max_features='sqrt', criterion = 'entropy', random_state = 42)\n","\n","rf_base_cv.fit(X_dev_cntvect_df,y_dev)"],"execution_count":11,"outputs":[{"output_type":"execute_result","data":{"text/plain":["RandomForestClassifier(bootstrap=True, ccp_alpha=0.0, class_weight=None,\n","                       criterion='entropy', max_depth=None, max_features='sqrt',\n","                       max_leaf_nodes=None, max_samples=None,\n","                       min_impurity_decrease=0.0, min_impurity_split=None,\n","                       min_samples_leaf=1, min_samples_split=2,\n","                       min_weight_fraction_leaf=0.0, n_estimators=10,\n","                       n_jobs=None, oob_score=False, random_state=42, verbose=0,\n","                       warm_start=False)"]},"metadata":{"tags":[]},"execution_count":11}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"1Nx7peeYs2iD","executionInfo":{"status":"ok","timestamp":1621714695313,"user_tz":360,"elapsed":1690,"user":{"displayName":"Laura Elliott","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiR_lgn2QjRHbMFy-LVQ-MZiwLY0GVJVCQPQcKj=s64","userId":"17671003486597139296"}},"outputId":"6c906dca-fa72-4b64-cf2e-00074511d46d"},"source":["'''\n","3.1c Print scores - AUC score for both dev and cv sets; very high train score - significant overfitting and val auc score of .64\n","'''\n","y_cntvect_prob_dev = rf_base_cv.predict_proba(X_dev_cntvect_df)\n","roc_auc_cntvect_dev = roc_auc_score(y_dev, y_cntvect_prob_dev, multi_class=\"ovo\",\n","                                  average=\"macro\")\n","\n","print(\"CntVect RF Train AUC Score:\", round(roc_auc_cntvect_dev,4))\n","\n","y_cntvect_prob_val = rf_base_cv.predict_proba(X_cv_cntvect_df)\n","roc_auc_cntvect_val = roc_auc_score(y_cv, y_cntvect_prob_val, multi_class=\"ovo\",\n","                                  average=\"macro\")\n","print(\"CntVect RF Val AUC Score:   \", round(roc_auc_cntvect_val,4))\n"],"execution_count":12,"outputs":[{"output_type":"stream","text":["CntVect RF Train AUC Score: 0.9999\n","CntVect RF Val AUC Score:    0.6557\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"7zT80TVmK_8Z","executionInfo":{"status":"ok","timestamp":1621714819593,"user_tz":360,"elapsed":120566,"user":{"displayName":"Laura Elliott","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiR_lgn2QjRHbMFy-LVQ-MZiwLY0GVJVCQPQcKj=s64","userId":"17671003486597139296"}},"outputId":"e6c9cc08-9994-42bb-c7c1-71560995c5b4"},"source":["'''\n","3.1d. Running base random forest model with more estimators\n","'''\n","rf_base_cv_1000  = RandomForestClassifier(n_estimators=1000, max_features='sqrt', criterion = 'entropy', random_state = 42)\n","\n","rf_base_cv_1000 .fit(X_dev_cntvect_df,y_dev)"],"execution_count":13,"outputs":[{"output_type":"execute_result","data":{"text/plain":["RandomForestClassifier(bootstrap=True, ccp_alpha=0.0, class_weight=None,\n","                       criterion='entropy', max_depth=None, max_features='sqrt',\n","                       max_leaf_nodes=None, max_samples=None,\n","                       min_impurity_decrease=0.0, min_impurity_split=None,\n","                       min_samples_leaf=1, min_samples_split=2,\n","                       min_weight_fraction_leaf=0.0, n_estimators=1000,\n","                       n_jobs=None, oob_score=False, random_state=42, verbose=0,\n","                       warm_start=False)"]},"metadata":{"tags":[]},"execution_count":13}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"gxJuS_JqLMdr","executionInfo":{"status":"ok","timestamp":1621714853531,"user_tz":360,"elapsed":14143,"user":{"displayName":"Laura Elliott","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiR_lgn2QjRHbMFy-LVQ-MZiwLY0GVJVCQPQcKj=s64","userId":"17671003486597139296"}},"outputId":"7ae480e1-a205-4179-c432-ef7b9aa97c2c"},"source":["'''\n","3.1e Print scores - AUC score for both dev and cv sets; very high train score - still significant overfitting but increasing estimators improves val score significantly\n","'''\n","y_cntvect_prob_dev_1000 = rf_base_cv_1000.predict_proba(X_dev_cntvect_df)\n","roc_auc_cntvect_dev_1000 = roc_auc_score(y_dev, y_cntvect_prob_dev_1000, multi_class=\"ovo\",\n","                                  average=\"macro\")\n","\n","print(\"CntVect RF Train AUC Score:\", round(roc_auc_cntvect_dev_1000,4))\n","\n","y_cntvect_prob_val_1000 = rf_base_cv_1000.predict_proba(X_cv_cntvect_df)\n","roc_auc_cntvect_val_1000 = roc_auc_score(y_cv, y_cntvect_prob_val_1000, multi_class=\"ovo\",\n","                                  average=\"macro\")\n","print(\"CntVect RF Val AUC Score:   \", round(roc_auc_cntvect_val_1000,4))"],"execution_count":14,"outputs":[{"output_type":"stream","text":["CntVect RF Train AUC Score: 1.0\n","CntVect RF Val AUC Score:    0.8627\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"MXoFQ3hXtcEk","executionInfo":{"status":"ok","timestamp":1621714858857,"user_tz":360,"elapsed":1502,"user":{"displayName":"Laura Elliott","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiR_lgn2QjRHbMFy-LVQ-MZiwLY0GVJVCQPQcKj=s64","userId":"17671003486597139296"}},"outputId":"43ca2a6c-6c08-4b82-b159-a8b5ffe2cb00"},"source":["'''\n","3.2a Run tfidf with optimal params\n","'''\n","tfidf_vect = TfidfVectorizer(min_df=.001,max_df=.999, ngram_range=(1, 1))\n","X_dev_tfidf_df = pd.DataFrame(tfidf_vect.fit_transform(X_dev.BodyText_Clean).toarray(), index=X_dev.index, columns=tfidf_vect.get_feature_names())\n","print(X_dev_tfidf_df.shape)\n","X_dev_tfidf_df.head()\n","X_cv_tfidf = tfidf_vect.transform(X_cv.BodyText_Clean)"],"execution_count":15,"outputs":[{"output_type":"stream","text":["(11921, 3144)\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"Fln49R0svb9J","executionInfo":{"status":"ok","timestamp":1621714864859,"user_tz":360,"elapsed":3513,"user":{"displayName":"Laura Elliott","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiR_lgn2QjRHbMFy-LVQ-MZiwLY0GVJVCQPQcKj=s64","userId":"17671003486597139296"}},"outputId":"d2547e64-343d-49da-c2e2-7c6a050ccaa2"},"source":["'''\n","3.2b Running base random forest model\n","'''\n","rf_base_tfidf  = RandomForestClassifier(n_estimators=10, max_features='sqrt', criterion = 'entropy', random_state = 42)\n","\n","rf_base_tfidf.fit(X_dev_tfidf_df,y_dev)"],"execution_count":16,"outputs":[{"output_type":"execute_result","data":{"text/plain":["RandomForestClassifier(bootstrap=True, ccp_alpha=0.0, class_weight=None,\n","                       criterion='entropy', max_depth=None, max_features='sqrt',\n","                       max_leaf_nodes=None, max_samples=None,\n","                       min_impurity_decrease=0.0, min_impurity_split=None,\n","                       min_samples_leaf=1, min_samples_split=2,\n","                       min_weight_fraction_leaf=0.0, n_estimators=10,\n","                       n_jobs=None, oob_score=False, random_state=42, verbose=0,\n","                       warm_start=False)"]},"metadata":{"tags":[]},"execution_count":16}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"K-UQ6c3rxnpS","executionInfo":{"status":"ok","timestamp":1621714869711,"user_tz":360,"elapsed":1572,"user":{"displayName":"Laura Elliott","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiR_lgn2QjRHbMFy-LVQ-MZiwLY0GVJVCQPQcKj=s64","userId":"17671003486597139296"}},"outputId":"38a80f01-9375-4201-a883-1c4e818a9b63"},"source":["'''\n","3.2c Print scores - AUC score for both dev and cv sets; very high train score - significant overfitting again; but val scores are slightly higher than with count vectorizer\n","'''\n","y_tfidf_prob_dev = rf_base_tfidf.predict_proba(X_dev_tfidf_df)\n","roc_auc_tfidf_dev = roc_auc_score(y_dev, y_tfidf_prob_dev, multi_class=\"ovo\",\n","                                  average=\"macro\")\n","\n","print(\"TFIDF RF Train AUC Score:\", round(roc_auc_tfidf_dev,4))\n","\n","y_tfidf_prob_val = rf_base_tfidf.predict_proba(X_cv_tfidf)\n","roc_auc_tfidf_val = roc_auc_score(y_cv, y_tfidf_prob_val, multi_class=\"ovo\",\n","                                  average=\"macro\")\n","print(\"TFIDF RF Val AUC Score:  \", round(roc_auc_tfidf_val,4))"],"execution_count":17,"outputs":[{"output_type":"stream","text":["TFIDF RF Train AUC Score: 0.9999\n","TFIDF RF Val AUC Score:   0.6285\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"cup5wkjXOcWV","executionInfo":{"status":"ok","timestamp":1621715194175,"user_tz":360,"elapsed":319289,"user":{"displayName":"Laura Elliott","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiR_lgn2QjRHbMFy-LVQ-MZiwLY0GVJVCQPQcKj=s64","userId":"17671003486597139296"}},"outputId":"cffb1f7b-0b1a-493b-d54d-b08389e82a0e"},"source":["'''\n","3.2d. Running base random forest model with more estimators\n","'''\n","rf_base_tfidf_1000  = RandomForestClassifier(n_estimators=1000, max_features='sqrt', criterion = 'entropy', random_state = 42)\n","\n","rf_base_tfidf_1000.fit(X_dev_tfidf_df,y_dev)"],"execution_count":18,"outputs":[{"output_type":"execute_result","data":{"text/plain":["RandomForestClassifier(bootstrap=True, ccp_alpha=0.0, class_weight=None,\n","                       criterion='entropy', max_depth=None, max_features='sqrt',\n","                       max_leaf_nodes=None, max_samples=None,\n","                       min_impurity_decrease=0.0, min_impurity_split=None,\n","                       min_samples_leaf=1, min_samples_split=2,\n","                       min_weight_fraction_leaf=0.0, n_estimators=1000,\n","                       n_jobs=None, oob_score=False, random_state=42, verbose=0,\n","                       warm_start=False)"]},"metadata":{"tags":[]},"execution_count":18}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"oAr20p18SCHm","executionInfo":{"status":"ok","timestamp":1621715222952,"user_tz":360,"elapsed":14742,"user":{"displayName":"Laura Elliott","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiR_lgn2QjRHbMFy-LVQ-MZiwLY0GVJVCQPQcKj=s64","userId":"17671003486597139296"}},"outputId":"5c262141-48dd-43a1-e1ac-289e17ed9189"},"source":["'''\n","3.2e Print scores - AUC score for both dev and cv sets; very high train score - still significant overfitting but increasing estimators improves val score significantly\n","this is best score so far\n","'''\n","y_tfidf_prob_dev_1000 = rf_base_tfidf_1000.predict_proba(X_dev_tfidf_df)\n","roc_auc_tfidf_dev_1000 = roc_auc_score(y_dev, y_tfidf_prob_dev_1000, multi_class=\"ovo\",\n","                                  average=\"macro\")\n","\n","print(\"TFIDF RF Train AUC Score:\", round(roc_auc_tfidf_dev_1000,4))\n","\n","y_tfidf_prob_val_1000 = rf_base_tfidf_1000.predict_proba(X_cv_tfidf)\n","roc_auc_tfidf_val_1000 = roc_auc_score(y_cv, y_tfidf_prob_val_1000, multi_class=\"ovo\",\n","                                  average=\"macro\")\n","print(\"TFIDF RF Val AUC Score:   \", round(roc_auc_tfidf_val_1000,4))"],"execution_count":19,"outputs":[{"output_type":"stream","text":["TFIDF RF Train AUC Score: 1.0\n","TFIDF RF Val AUC Score:    0.8678\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"XsQOCZ8d7h_O","executionInfo":{"status":"ok","timestamp":1621715295931,"user_tz":360,"elapsed":67398,"user":{"displayName":"Laura Elliott","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiR_lgn2QjRHbMFy-LVQ-MZiwLY0GVJVCQPQcKj=s64","userId":"17671003486597139296"}},"outputId":"93055ea3-64d1-4904-c91f-d02bbbaf808b"},"source":["'''\n","3.3a. Loading hypertuned doc2vec model from previous notebook and re-creating the tagged docs, then evaluate\n","'''\n","fnl_d2v_model_200 = Doc2Vec.load(\"/content/drive/My Drive/Capstone2/Data/final_d2v_500.model\")\n","\n","# Build separate dataframes with the dev, cv, and test \n","reversefactor = dict(zip(range(34),definitions))\n","\n","y_dev_rf = np.vectorize(reversefactor.get)(y_dev)\n","y_cv_rf = np.vectorize(reversefactor.get)(y_cv)\n","y_test_rf = np.vectorize(reversefactor.get)(y_test)\n","\n","d2v_dev_df = pd.DataFrame({'y': y_dev_rf, 'X': X_dev.BodyText_Clean})\n","d2v_cv_df = pd.DataFrame({'y': y_cv_rf, 'X': X_cv.BodyText_Clean})\n","d2v_test_df = pd.DataFrame({'y': y_test_rf, 'X': X_test.BodyText_Clean})\n","#d2v_dev_df.head()\n","\n","# Ensure they're all string datatype\n","d2v_dev_df['X'] = d2v_dev_df['X'].astype(str)\n","d2v_cv_df['X'] = d2v_cv_df['X'].astype(str)\n","d2v_test_df['X'] = d2v_test_df['X'].astype(str)\n","d2v_dev_df.head()\n","\n","class TaggedDocumentIterator(object):\n","    def __init__(self, doc_list, labels_list):\n","        self.labels_list = labels_list\n","        self.doc_list = doc_list\n","    def __iter__(self):\n","        for idx, doc in enumerate(self.doc_list):\n","            yield TaggedDocument(words=doc.split(), tags=[self.labels_list[idx]])\n"," \n","docLabels_dev = list(d2v_dev_df['y'])\n","data_dev = list(d2v_dev_df['X'])\n","tagged_docs_dev = TaggedDocumentIterator(data_dev, docLabels_dev)\n","\n","docLabels_cv = list(d2v_cv_df['y'])\n","data_cv = list(d2v_cv_df['X'])\n","tagged_docs_cv = TaggedDocumentIterator(data_cv, docLabels_cv)\n","\n","docLabels_test = list(d2v_test_df['y'])\n","data_test = list(d2v_test_df['X'])\n","tagged_docs_test = TaggedDocumentIterator(data_test, docLabels_test)\n","\n","type(tagged_docs_dev)\n","d2v_dev_df.info()\n","\n","fnl_dev_targets, fnl_dev_regressors = zip(*[(doc.tags[0], fnl_d2v_model_200.infer_vector(doc.words, steps=20)) for doc in tagged_docs_dev])\n","fnl_cv_targets, fnl_cv_regressors = zip(*[(doc.tags[0], fnl_d2v_model_200.infer_vector(doc.words, steps=20)) for doc in tagged_docs_cv])"],"execution_count":20,"outputs":[{"output_type":"stream","text":["<class 'pandas.core.frame.DataFrame'>\n","Int64Index: 11921 entries, 8286 to 3207\n","Data columns (total 2 columns):\n"," #   Column  Non-Null Count  Dtype \n","---  ------  --------------  ----- \n"," 0   y       11921 non-null  object\n"," 1   X       11921 non-null  object\n","dtypes: object(2)\n","memory usage: 279.4+ KB\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"7CO14MnTzBSU","executionInfo":{"status":"ok","timestamp":1621715320090,"user_tz":360,"elapsed":18798,"user":{"displayName":"Laura Elliott","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiR_lgn2QjRHbMFy-LVQ-MZiwLY0GVJVCQPQcKj=s64","userId":"17671003486597139296"}}},"source":["'''\n","3.3b. Run base random forest multi-class model to evaluate the success of doc2vec transform on cv set\n","'''\n","\n","fnl_rf_base_d2v  = RandomForestClassifier(n_estimators=10, max_features='sqrt', criterion = 'entropy', random_state = 42)\n","fnl_fit = fnl_rf_base_d2v.fit(fnl_dev_regressors, fnl_dev_targets)\n","fnl_cv_targets_pred_d2v = fnl_rf_base_d2v.predict(fnl_cv_regressors)\n"],"execution_count":21,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"5dySjQ0KzV7d","executionInfo":{"status":"ok","timestamp":1621715325526,"user_tz":360,"elapsed":1648,"user":{"displayName":"Laura Elliott","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiR_lgn2QjRHbMFy-LVQ-MZiwLY0GVJVCQPQcKj=s64","userId":"17671003486597139296"}},"outputId":"2bec432d-468c-4d5b-a567-b3927927a0e7"},"source":["'''\n","3.3c Print scores\n","'''\n","fnl_dev_targets_prob_d2v = fnl_rf_base_d2v.predict_proba(fnl_dev_regressors)\n","roc_auc_d2v_dev = roc_auc_score(fnl_dev_targets, fnl_dev_targets_prob_d2v, multi_class=\"ovo\",\n","                                  average=\"macro\")\n","\n","print(\"Doc2Vec RF Train AUC Score:\", round(roc_auc_d2v_dev,4))\n","\n","fnl_val_targets_prob_d2v = fnl_rf_base_d2v.predict_proba(fnl_cv_regressors)\n","#print(fnl_cv_targets_prob_d2v)\n","roc_auc_d2v_val = roc_auc_score(fnl_cv_targets, fnl_val_targets_prob_d2v, multi_class=\"ovo\", average=\"macro\")\n","\n","print(\"Cross Val AUC Score:        \", round(roc_auc_d2v_val,4))"],"execution_count":22,"outputs":[{"output_type":"stream","text":["Doc2Vec RF Train AUC Score: 1.0\n","Cross Val AUC Score:         0.6724\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"z2FTugBeUuot","executionInfo":{"status":"ok","timestamp":1621717194227,"user_tz":360,"elapsed":1865204,"user":{"displayName":"Laura Elliott","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiR_lgn2QjRHbMFy-LVQ-MZiwLY0GVJVCQPQcKj=s64","userId":"17671003486597139296"}}},"source":["'''\n","3.3d. Run base random forest multi-class model to evaluate the success of doc2vec transform on cv set - increasing estimators\n","'''\n","\n","fnl_rf_base_d2v_1000  = RandomForestClassifier(n_estimators=1000, max_features='sqrt', criterion = 'entropy', random_state = 42)\n","fnl_fit_1000 = fnl_rf_base_d2v_1000 .fit(fnl_dev_regressors, fnl_dev_targets)\n","fnl_cv_targets_pred_d2v_1000 = fnl_rf_base_d2v_1000.predict(fnl_cv_regressors)"],"execution_count":23,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"MGsgQ0C-U-C_","executionInfo":{"status":"ok","timestamp":1621717417483,"user_tz":360,"elapsed":10707,"user":{"displayName":"Laura Elliott","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiR_lgn2QjRHbMFy-LVQ-MZiwLY0GVJVCQPQcKj=s64","userId":"17671003486597139296"}},"outputId":"39f32c0c-a525-4eec-d3d4-a29fe8036843"},"source":["'''\n","3.3e Print scores\n","'''\n","fnl_dev_targets_prob_d2v_1000 = fnl_rf_base_d2v_1000.predict_proba(fnl_dev_regressors)\n","roc_auc_d2v_dev_1000 = roc_auc_score(fnl_dev_targets, fnl_dev_targets_prob_d2v_1000, multi_class=\"ovo\",\n","                                  average=\"macro\")\n","\n","print(\"Doc2Vec RF Train AUC Score:\", round(roc_auc_d2v_dev_1000,4))\n","\n","fnl_val_targets_prob_d2v_1000 = fnl_rf_base_d2v_1000.predict_proba(fnl_cv_regressors)\n","#print(fnl_cv_targets_prob_d2v)\n","roc_auc_d2v_val_1000 = roc_auc_score(fnl_cv_targets, fnl_val_targets_prob_d2v_1000, multi_class=\"ovo\", average=\"macro\")\n","\n","print(\"Cross Val AUC Score:        \", round(roc_auc_d2v_val_1000,4))"],"execution_count":24,"outputs":[{"output_type":"stream","text":["Doc2Vec RF Train AUC Score: 1.0\n","Cross Val AUC Score:         0.8574\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"2qRwIp9PcTMC","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1621717449500,"user_tz":360,"elapsed":176,"user":{"displayName":"Laura Elliott","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiR_lgn2QjRHbMFy-LVQ-MZiwLY0GVJVCQPQcKj=s64","userId":"17671003486597139296"}},"outputId":"effb0fdd-d638-4b21-a164-a345ce6c1153"},"source":["'''\n","Code for creating a nice table for model comparison results\n","'''\n","x = PrettyTable()\n","x.field_names = [\"Model\",\"Estimators\",\"Transformation\",'AUC Train Score' ,'AUC Val Score']\n","\n","\n","x.add_row([\"Random Forest\",1000, \"Tf-idf\",round(roc_auc_tfidf_dev_1000,4),round(roc_auc_tfidf_val_1000,4)])\n","x.add_row([\"Random Forest\",1000, \"Count Vectorizer\", round(roc_auc_cntvect_dev_1000,4),round(roc_auc_cntvect_val_1000,4)])\n","x.add_row([\"Random Forest\",1000, \"Doc2Vec\", round(roc_auc_d2v_dev_1000,4), round(roc_auc_d2v_val_1000,4)])\n","x.add_row([\"Random Forest\",10, \"Count Vectorizer\", round(roc_auc_cntvect_dev,4),round(roc_auc_cntvect_val,4)])\n","x.add_row([\"Random Forest\",10, \"Doc2Vec\", round(roc_auc_d2v_dev,4), round(roc_auc_d2v_val,4)])\n","x.add_row([\"Random Forest\",10, \"Tf-idf\", round(roc_auc_tfidf_dev,4),round(roc_auc_tfidf_val,4)])\n","\n","print(x)\n"],"execution_count":26,"outputs":[{"output_type":"stream","text":["+---------------+------------+------------------+-----------------+---------------+\n","|     Model     | Estimators |  Transformation  | AUC Train Score | AUC Val Score |\n","+---------------+------------+------------------+-----------------+---------------+\n","| Random Forest |    1000    |      Tf-idf      |       1.0       |     0.8678    |\n","| Random Forest |    1000    | Count Vectorizer |       1.0       |     0.8627    |\n","| Random Forest |    1000    |     Doc2Vec      |       1.0       |     0.8574    |\n","| Random Forest |     10     | Count Vectorizer |      0.9999     |     0.6557    |\n","| Random Forest |     10     |     Doc2Vec      |       1.0       |     0.6724    |\n","| Random Forest |     10     |      Tf-idf      |      0.9999     |     0.6285    |\n","+---------------+------------+------------------+-----------------+---------------+\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"d0rTcIJX1JNc"},"source":["# Task 4 - hypertune to reduce overfitting"]},{"cell_type":"code","metadata":{"id":"a5l-v_aCbHBR"},"source":["'''\n","4.1 Grid Search 5 fold CV to see if we can reduce overfitting\n","parameters tested will be \n","max_features = ['sqrt', .25, .50] of the number of features - not included as this gave an error\n","min_samples_leaf = [1,2,5] branches will stop splitting once the leaves have that number of samples default is 1\n","mtry(number of features randomly selected for each tree) - did not find this parameter as an option (same as max_samples?)\n","max_depth = [None, 5,10,15] (how deep the tree can growand reduction of this parameter will reduce complexity) default is None\n","n_estimators = [100,500,1000,2000] default is 100 - best param was 100\n","'''\n","# Create the parameter grid \n","param_grid = {\n","    'n_estimators': [10,100,500,1000],\n","    'min_samples_leaf' : [1,2,5],\n","    'max_depth' : [None, 5,10,15]\n","}\n","# Create a based model\n","rf = RandomForestClassifier(max_features='sqrt', criterion = 'entropy', random_state = 42)\n","# Instantiate the grid search model\n","grid_search = GridSearchCV(estimator = rf, param_grid = param_grid, scoring= 'roc_auc_ovo', cv = 5, verbose = 2)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"RbyQIjoLsHb2","executionInfo":{"status":"ok","timestamp":1620893791540,"user_tz":360,"elapsed":14075768,"user":{"displayName":"Laura Elliott","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiR_lgn2QjRHbMFy-LVQ-MZiwLY0GVJVCQPQcKj=s64","userId":"17671003486597139296"}},"outputId":"b4c73253-8f40-41c0-a166-0d81bb29df23"},"source":["# 4.2 Fit the grid search to the tfidf trained data since this provided the best accuracy of all 3 transformations\n","grid_search.fit(X_dev_tfidf_df,y_dev)\n","best_grid = grid_search.best_estimator_\n","\n","print(grid_search.best_params_)\n","print(best_grid)\n","print(grid_search.cv_results_)\n"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Fitting 5 folds for each of 48 candidates, totalling 240 fits\n","[CV] max_depth=None, min_samples_leaf=1, n_estimators=10 .............\n"],"name":"stdout"},{"output_type":"stream","text":["[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"],"name":"stderr"},{"output_type":"stream","text":["[CV]  max_depth=None, min_samples_leaf=1, n_estimators=10, total=   4.2s\n","[CV] max_depth=None, min_samples_leaf=1, n_estimators=10 .............\n"],"name":"stdout"},{"output_type":"stream","text":["[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    4.2s remaining:    0.0s\n"],"name":"stderr"},{"output_type":"stream","text":["[CV]  max_depth=None, min_samples_leaf=1, n_estimators=10, total=   3.6s\n","[CV] max_depth=None, min_samples_leaf=1, n_estimators=10 .............\n","[CV]  max_depth=None, min_samples_leaf=1, n_estimators=10, total=   3.7s\n","[CV] max_depth=None, min_samples_leaf=1, n_estimators=10 .............\n","[CV]  max_depth=None, min_samples_leaf=1, n_estimators=10, total=   3.6s\n","[CV] max_depth=None, min_samples_leaf=1, n_estimators=10 .............\n","[CV]  max_depth=None, min_samples_leaf=1, n_estimators=10, total=   3.7s\n","[CV] max_depth=None, min_samples_leaf=1, n_estimators=100 ............\n","[CV]  max_depth=None, min_samples_leaf=1, n_estimators=100, total=  30.6s\n","[CV] max_depth=None, min_samples_leaf=1, n_estimators=100 ............\n","[CV]  max_depth=None, min_samples_leaf=1, n_estimators=100, total=  30.3s\n","[CV] max_depth=None, min_samples_leaf=1, n_estimators=100 ............\n","[CV]  max_depth=None, min_samples_leaf=1, n_estimators=100, total=  30.6s\n","[CV] max_depth=None, min_samples_leaf=1, n_estimators=100 ............\n","[CV]  max_depth=None, min_samples_leaf=1, n_estimators=100, total=  30.7s\n","[CV] max_depth=None, min_samples_leaf=1, n_estimators=100 ............\n","[CV]  max_depth=None, min_samples_leaf=1, n_estimators=100, total=  30.5s\n","[CV] max_depth=None, min_samples_leaf=1, n_estimators=500 ............\n","[CV]  max_depth=None, min_samples_leaf=1, n_estimators=500, total= 2.6min\n","[CV] max_depth=None, min_samples_leaf=1, n_estimators=500 ............\n","[CV]  max_depth=None, min_samples_leaf=1, n_estimators=500, total= 2.5min\n","[CV] max_depth=None, min_samples_leaf=1, n_estimators=500 ............\n","[CV]  max_depth=None, min_samples_leaf=1, n_estimators=500, total= 2.5min\n","[CV] max_depth=None, min_samples_leaf=1, n_estimators=500 ............\n","[CV]  max_depth=None, min_samples_leaf=1, n_estimators=500, total= 2.5min\n","[CV] max_depth=None, min_samples_leaf=1, n_estimators=500 ............\n","[CV]  max_depth=None, min_samples_leaf=1, n_estimators=500, total= 2.5min\n","[CV] max_depth=None, min_samples_leaf=1, n_estimators=1000 ...........\n","[CV]  max_depth=None, min_samples_leaf=1, n_estimators=1000, total= 4.9min\n","[CV] max_depth=None, min_samples_leaf=1, n_estimators=1000 ...........\n","[CV]  max_depth=None, min_samples_leaf=1, n_estimators=1000, total= 4.5min\n","[CV] max_depth=None, min_samples_leaf=1, n_estimators=1000 ...........\n","[CV]  max_depth=None, min_samples_leaf=1, n_estimators=1000, total= 4.4min\n","[CV] max_depth=None, min_samples_leaf=1, n_estimators=1000 ...........\n","[CV]  max_depth=None, min_samples_leaf=1, n_estimators=1000, total= 4.5min\n","[CV] max_depth=None, min_samples_leaf=1, n_estimators=1000 ...........\n","[CV]  max_depth=None, min_samples_leaf=1, n_estimators=1000, total= 4.4min\n","[CV] max_depth=None, min_samples_leaf=2, n_estimators=10 .............\n","[CV]  max_depth=None, min_samples_leaf=2, n_estimators=10, total=   2.9s\n","[CV] max_depth=None, min_samples_leaf=2, n_estimators=10 .............\n","[CV]  max_depth=None, min_samples_leaf=2, n_estimators=10, total=   2.9s\n","[CV] max_depth=None, min_samples_leaf=2, n_estimators=10 .............\n","[CV]  max_depth=None, min_samples_leaf=2, n_estimators=10, total=   3.0s\n","[CV] max_depth=None, min_samples_leaf=2, n_estimators=10 .............\n","[CV]  max_depth=None, min_samples_leaf=2, n_estimators=10, total=   2.9s\n","[CV] max_depth=None, min_samples_leaf=2, n_estimators=10 .............\n","[CV]  max_depth=None, min_samples_leaf=2, n_estimators=10, total=   2.9s\n","[CV] max_depth=None, min_samples_leaf=2, n_estimators=100 ............\n","[CV]  max_depth=None, min_samples_leaf=2, n_estimators=100, total=  23.3s\n","[CV] max_depth=None, min_samples_leaf=2, n_estimators=100 ............\n","[CV]  max_depth=None, min_samples_leaf=2, n_estimators=100, total=  23.0s\n","[CV] max_depth=None, min_samples_leaf=2, n_estimators=100 ............\n","[CV]  max_depth=None, min_samples_leaf=2, n_estimators=100, total=  23.3s\n","[CV] max_depth=None, min_samples_leaf=2, n_estimators=100 ............\n","[CV]  max_depth=None, min_samples_leaf=2, n_estimators=100, total=  23.6s\n","[CV] max_depth=None, min_samples_leaf=2, n_estimators=100 ............\n","[CV]  max_depth=None, min_samples_leaf=2, n_estimators=100, total=  22.9s\n","[CV] max_depth=None, min_samples_leaf=2, n_estimators=500 ............\n","[CV]  max_depth=None, min_samples_leaf=2, n_estimators=500, total= 1.9min\n","[CV] max_depth=None, min_samples_leaf=2, n_estimators=500 ............\n","[CV]  max_depth=None, min_samples_leaf=2, n_estimators=500, total= 2.1min\n","[CV] max_depth=None, min_samples_leaf=2, n_estimators=500 ............\n","[CV]  max_depth=None, min_samples_leaf=2, n_estimators=500, total= 2.0min\n","[CV] max_depth=None, min_samples_leaf=2, n_estimators=500 ............\n","[CV]  max_depth=None, min_samples_leaf=2, n_estimators=500, total= 2.0min\n","[CV] max_depth=None, min_samples_leaf=2, n_estimators=500 ............\n","[CV]  max_depth=None, min_samples_leaf=2, n_estimators=500, total= 2.0min\n","[CV] max_depth=None, min_samples_leaf=2, n_estimators=1000 ...........\n","[CV]  max_depth=None, min_samples_leaf=2, n_estimators=1000, total= 3.8min\n","[CV] max_depth=None, min_samples_leaf=2, n_estimators=1000 ...........\n","[CV]  max_depth=None, min_samples_leaf=2, n_estimators=1000, total= 4.0min\n","[CV] max_depth=None, min_samples_leaf=2, n_estimators=1000 ...........\n","[CV]  max_depth=None, min_samples_leaf=2, n_estimators=1000, total= 4.5min\n","[CV] max_depth=None, min_samples_leaf=2, n_estimators=1000 ...........\n","[CV]  max_depth=None, min_samples_leaf=2, n_estimators=1000, total= 4.1min\n","[CV] max_depth=None, min_samples_leaf=2, n_estimators=1000 ...........\n","[CV]  max_depth=None, min_samples_leaf=2, n_estimators=1000, total= 4.1min\n","[CV] max_depth=None, min_samples_leaf=5, n_estimators=10 .............\n","[CV]  max_depth=None, min_samples_leaf=5, n_estimators=10, total=   2.8s\n","[CV] max_depth=None, min_samples_leaf=5, n_estimators=10 .............\n","[CV]  max_depth=None, min_samples_leaf=5, n_estimators=10, total=   2.7s\n","[CV] max_depth=None, min_samples_leaf=5, n_estimators=10 .............\n","[CV]  max_depth=None, min_samples_leaf=5, n_estimators=10, total=   2.9s\n","[CV] max_depth=None, min_samples_leaf=5, n_estimators=10 .............\n","[CV]  max_depth=None, min_samples_leaf=5, n_estimators=10, total=   2.7s\n","[CV] max_depth=None, min_samples_leaf=5, n_estimators=10 .............\n","[CV]  max_depth=None, min_samples_leaf=5, n_estimators=10, total=   2.8s\n","[CV] max_depth=None, min_samples_leaf=5, n_estimators=100 ............\n","[CV]  max_depth=None, min_samples_leaf=5, n_estimators=100, total=  21.3s\n","[CV] max_depth=None, min_samples_leaf=5, n_estimators=100 ............\n","[CV]  max_depth=None, min_samples_leaf=5, n_estimators=100, total=  21.1s\n","[CV] max_depth=None, min_samples_leaf=5, n_estimators=100 ............\n","[CV]  max_depth=None, min_samples_leaf=5, n_estimators=100, total=  21.6s\n","[CV] max_depth=None, min_samples_leaf=5, n_estimators=100 ............\n","[CV]  max_depth=None, min_samples_leaf=5, n_estimators=100, total=  21.7s\n","[CV] max_depth=None, min_samples_leaf=5, n_estimators=100 ............\n","[CV]  max_depth=None, min_samples_leaf=5, n_estimators=100, total=  21.5s\n","[CV] max_depth=None, min_samples_leaf=5, n_estimators=500 ............\n","[CV]  max_depth=None, min_samples_leaf=5, n_estimators=500, total= 1.8min\n","[CV] max_depth=None, min_samples_leaf=5, n_estimators=500 ............\n","[CV]  max_depth=None, min_samples_leaf=5, n_estimators=500, total= 1.7min\n","[CV] max_depth=None, min_samples_leaf=5, n_estimators=500 ............\n","[CV]  max_depth=None, min_samples_leaf=5, n_estimators=500, total= 1.8min\n","[CV] max_depth=None, min_samples_leaf=5, n_estimators=500 ............\n","[CV]  max_depth=None, min_samples_leaf=5, n_estimators=500, total= 1.8min\n","[CV] max_depth=None, min_samples_leaf=5, n_estimators=500 ............\n","[CV]  max_depth=None, min_samples_leaf=5, n_estimators=500, total= 1.8min\n","[CV] max_depth=None, min_samples_leaf=5, n_estimators=1000 ...........\n","[CV]  max_depth=None, min_samples_leaf=5, n_estimators=1000, total= 3.5min\n","[CV] max_depth=None, min_samples_leaf=5, n_estimators=1000 ...........\n","[CV]  max_depth=None, min_samples_leaf=5, n_estimators=1000, total= 3.3min\n","[CV] max_depth=None, min_samples_leaf=5, n_estimators=1000 ...........\n","[CV]  max_depth=None, min_samples_leaf=5, n_estimators=1000, total= 3.2min\n","[CV] max_depth=None, min_samples_leaf=5, n_estimators=1000 ...........\n","[CV]  max_depth=None, min_samples_leaf=5, n_estimators=1000, total= 3.3min\n","[CV] max_depth=None, min_samples_leaf=5, n_estimators=1000 ...........\n","[CV]  max_depth=None, min_samples_leaf=5, n_estimators=1000, total= 3.3min\n","[CV] max_depth=5, min_samples_leaf=1, n_estimators=10 ................\n","[CV] . max_depth=5, min_samples_leaf=1, n_estimators=10, total=   1.5s\n","[CV] max_depth=5, min_samples_leaf=1, n_estimators=10 ................\n","[CV] . max_depth=5, min_samples_leaf=1, n_estimators=10, total=   2.0s\n","[CV] max_depth=5, min_samples_leaf=1, n_estimators=10 ................\n","[CV] . max_depth=5, min_samples_leaf=1, n_estimators=10, total=   1.3s\n","[CV] max_depth=5, min_samples_leaf=1, n_estimators=10 ................\n","[CV] . max_depth=5, min_samples_leaf=1, n_estimators=10, total=   1.3s\n","[CV] max_depth=5, min_samples_leaf=1, n_estimators=10 ................\n","[CV] . max_depth=5, min_samples_leaf=1, n_estimators=10, total=   1.4s\n","[CV] max_depth=5, min_samples_leaf=1, n_estimators=100 ...............\n","[CV]  max_depth=5, min_samples_leaf=1, n_estimators=100, total=   6.9s\n","[CV] max_depth=5, min_samples_leaf=1, n_estimators=100 ...............\n","[CV]  max_depth=5, min_samples_leaf=1, n_estimators=100, total=   6.9s\n","[CV] max_depth=5, min_samples_leaf=1, n_estimators=100 ...............\n","[CV]  max_depth=5, min_samples_leaf=1, n_estimators=100, total=   6.9s\n","[CV] max_depth=5, min_samples_leaf=1, n_estimators=100 ...............\n","[CV]  max_depth=5, min_samples_leaf=1, n_estimators=100, total=   7.0s\n","[CV] max_depth=5, min_samples_leaf=1, n_estimators=100 ...............\n","[CV]  max_depth=5, min_samples_leaf=1, n_estimators=100, total=   6.9s\n","[CV] max_depth=5, min_samples_leaf=1, n_estimators=500 ...............\n","[CV]  max_depth=5, min_samples_leaf=1, n_estimators=500, total=  31.6s\n","[CV] max_depth=5, min_samples_leaf=1, n_estimators=500 ...............\n","[CV]  max_depth=5, min_samples_leaf=1, n_estimators=500, total=  31.4s\n","[CV] max_depth=5, min_samples_leaf=1, n_estimators=500 ...............\n","[CV]  max_depth=5, min_samples_leaf=1, n_estimators=500, total=  31.8s\n","[CV] max_depth=5, min_samples_leaf=1, n_estimators=500 ...............\n","[CV]  max_depth=5, min_samples_leaf=1, n_estimators=500, total=  31.5s\n","[CV] max_depth=5, min_samples_leaf=1, n_estimators=500 ...............\n","[CV]  max_depth=5, min_samples_leaf=1, n_estimators=500, total=  31.6s\n","[CV] max_depth=5, min_samples_leaf=1, n_estimators=1000 ..............\n","[CV]  max_depth=5, min_samples_leaf=1, n_estimators=1000, total= 1.0min\n","[CV] max_depth=5, min_samples_leaf=1, n_estimators=1000 ..............\n","[CV]  max_depth=5, min_samples_leaf=1, n_estimators=1000, total= 1.1min\n","[CV] max_depth=5, min_samples_leaf=1, n_estimators=1000 ..............\n","[CV]  max_depth=5, min_samples_leaf=1, n_estimators=1000, total= 1.0min\n","[CV] max_depth=5, min_samples_leaf=1, n_estimators=1000 ..............\n","[CV]  max_depth=5, min_samples_leaf=1, n_estimators=1000, total= 1.0min\n","[CV] max_depth=5, min_samples_leaf=1, n_estimators=1000 ..............\n","[CV]  max_depth=5, min_samples_leaf=1, n_estimators=1000, total= 1.1min\n","[CV] max_depth=5, min_samples_leaf=2, n_estimators=10 ................\n","[CV] . max_depth=5, min_samples_leaf=2, n_estimators=10, total=   1.4s\n","[CV] max_depth=5, min_samples_leaf=2, n_estimators=10 ................\n","[CV] . max_depth=5, min_samples_leaf=2, n_estimators=10, total=   1.4s\n","[CV] max_depth=5, min_samples_leaf=2, n_estimators=10 ................\n","[CV] . max_depth=5, min_samples_leaf=2, n_estimators=10, total=   1.4s\n","[CV] max_depth=5, min_samples_leaf=2, n_estimators=10 ................\n","[CV] . max_depth=5, min_samples_leaf=2, n_estimators=10, total=   1.4s\n","[CV] max_depth=5, min_samples_leaf=2, n_estimators=10 ................\n","[CV] . max_depth=5, min_samples_leaf=2, n_estimators=10, total=   1.4s\n","[CV] max_depth=5, min_samples_leaf=2, n_estimators=100 ...............\n","[CV]  max_depth=5, min_samples_leaf=2, n_estimators=100, total=   6.9s\n","[CV] max_depth=5, min_samples_leaf=2, n_estimators=100 ...............\n","[CV]  max_depth=5, min_samples_leaf=2, n_estimators=100, total=   6.9s\n","[CV] max_depth=5, min_samples_leaf=2, n_estimators=100 ...............\n","[CV]  max_depth=5, min_samples_leaf=2, n_estimators=100, total=   6.9s\n","[CV] max_depth=5, min_samples_leaf=2, n_estimators=100 ...............\n","[CV]  max_depth=5, min_samples_leaf=2, n_estimators=100, total=   6.9s\n","[CV] max_depth=5, min_samples_leaf=2, n_estimators=100 ...............\n","[CV]  max_depth=5, min_samples_leaf=2, n_estimators=100, total=   6.9s\n","[CV] max_depth=5, min_samples_leaf=2, n_estimators=500 ...............\n","[CV]  max_depth=5, min_samples_leaf=2, n_estimators=500, total=  31.4s\n","[CV] max_depth=5, min_samples_leaf=2, n_estimators=500 ...............\n","[CV]  max_depth=5, min_samples_leaf=2, n_estimators=500, total=  31.5s\n","[CV] max_depth=5, min_samples_leaf=2, n_estimators=500 ...............\n","[CV]  max_depth=5, min_samples_leaf=2, n_estimators=500, total=  31.5s\n","[CV] max_depth=5, min_samples_leaf=2, n_estimators=500 ...............\n","[CV]  max_depth=5, min_samples_leaf=2, n_estimators=500, total=  31.4s\n","[CV] max_depth=5, min_samples_leaf=2, n_estimators=500 ...............\n","[CV]  max_depth=5, min_samples_leaf=2, n_estimators=500, total=  30.9s\n","[CV] max_depth=5, min_samples_leaf=2, n_estimators=1000 ..............\n","[CV]  max_depth=5, min_samples_leaf=2, n_estimators=1000, total= 1.0min\n","[CV] max_depth=5, min_samples_leaf=2, n_estimators=1000 ..............\n","[CV]  max_depth=5, min_samples_leaf=2, n_estimators=1000, total= 1.0min\n","[CV] max_depth=5, min_samples_leaf=2, n_estimators=1000 ..............\n","[CV]  max_depth=5, min_samples_leaf=2, n_estimators=1000, total= 1.0min\n","[CV] max_depth=5, min_samples_leaf=2, n_estimators=1000 ..............\n","[CV]  max_depth=5, min_samples_leaf=2, n_estimators=1000, total= 1.0min\n","[CV] max_depth=5, min_samples_leaf=2, n_estimators=1000 ..............\n","[CV]  max_depth=5, min_samples_leaf=2, n_estimators=1000, total= 1.0min\n","[CV] max_depth=5, min_samples_leaf=5, n_estimators=10 ................\n","[CV] . max_depth=5, min_samples_leaf=5, n_estimators=10, total=   1.3s\n","[CV] max_depth=5, min_samples_leaf=5, n_estimators=10 ................\n","[CV] . max_depth=5, min_samples_leaf=5, n_estimators=10, total=   1.3s\n","[CV] max_depth=5, min_samples_leaf=5, n_estimators=10 ................\n","[CV] . max_depth=5, min_samples_leaf=5, n_estimators=10, total=   1.3s\n","[CV] max_depth=5, min_samples_leaf=5, n_estimators=10 ................\n","[CV] . max_depth=5, min_samples_leaf=5, n_estimators=10, total=   1.3s\n","[CV] max_depth=5, min_samples_leaf=5, n_estimators=10 ................\n","[CV] . max_depth=5, min_samples_leaf=5, n_estimators=10, total=   1.3s\n","[CV] max_depth=5, min_samples_leaf=5, n_estimators=100 ...............\n","[CV]  max_depth=5, min_samples_leaf=5, n_estimators=100, total=   6.6s\n","[CV] max_depth=5, min_samples_leaf=5, n_estimators=100 ...............\n","[CV]  max_depth=5, min_samples_leaf=5, n_estimators=100, total=   6.6s\n","[CV] max_depth=5, min_samples_leaf=5, n_estimators=100 ...............\n","[CV]  max_depth=5, min_samples_leaf=5, n_estimators=100, total=   6.7s\n","[CV] max_depth=5, min_samples_leaf=5, n_estimators=100 ...............\n","[CV]  max_depth=5, min_samples_leaf=5, n_estimators=100, total=   6.7s\n","[CV] max_depth=5, min_samples_leaf=5, n_estimators=100 ...............\n","[CV]  max_depth=5, min_samples_leaf=5, n_estimators=100, total=   6.7s\n","[CV] max_depth=5, min_samples_leaf=5, n_estimators=500 ...............\n","[CV]  max_depth=5, min_samples_leaf=5, n_estimators=500, total=  30.3s\n","[CV] max_depth=5, min_samples_leaf=5, n_estimators=500 ...............\n","[CV]  max_depth=5, min_samples_leaf=5, n_estimators=500, total=  30.3s\n","[CV] max_depth=5, min_samples_leaf=5, n_estimators=500 ...............\n","[CV]  max_depth=5, min_samples_leaf=5, n_estimators=500, total=  30.2s\n","[CV] max_depth=5, min_samples_leaf=5, n_estimators=500 ...............\n","[CV]  max_depth=5, min_samples_leaf=5, n_estimators=500, total=  30.4s\n","[CV] max_depth=5, min_samples_leaf=5, n_estimators=500 ...............\n","[CV]  max_depth=5, min_samples_leaf=5, n_estimators=500, total=  30.7s\n","[CV] max_depth=5, min_samples_leaf=5, n_estimators=1000 ..............\n","[CV]  max_depth=5, min_samples_leaf=5, n_estimators=1000, total=  59.9s\n","[CV] max_depth=5, min_samples_leaf=5, n_estimators=1000 ..............\n","[CV]  max_depth=5, min_samples_leaf=5, n_estimators=1000, total=  60.0s\n","[CV] max_depth=5, min_samples_leaf=5, n_estimators=1000 ..............\n","[CV]  max_depth=5, min_samples_leaf=5, n_estimators=1000, total=  59.9s\n","[CV] max_depth=5, min_samples_leaf=5, n_estimators=1000 ..............\n","[CV]  max_depth=5, min_samples_leaf=5, n_estimators=1000, total= 1.0min\n","[CV] max_depth=5, min_samples_leaf=5, n_estimators=1000 ..............\n","[CV]  max_depth=5, min_samples_leaf=5, n_estimators=1000, total= 1.0min\n","[CV] max_depth=10, min_samples_leaf=1, n_estimators=10 ...............\n","[CV]  max_depth=10, min_samples_leaf=1, n_estimators=10, total=   1.8s\n","[CV] max_depth=10, min_samples_leaf=1, n_estimators=10 ...............\n","[CV]  max_depth=10, min_samples_leaf=1, n_estimators=10, total=   1.8s\n","[CV] max_depth=10, min_samples_leaf=1, n_estimators=10 ...............\n","[CV]  max_depth=10, min_samples_leaf=1, n_estimators=10, total=   1.8s\n","[CV] max_depth=10, min_samples_leaf=1, n_estimators=10 ...............\n","[CV]  max_depth=10, min_samples_leaf=1, n_estimators=10, total=   1.8s\n","[CV] max_depth=10, min_samples_leaf=1, n_estimators=10 ...............\n","[CV]  max_depth=10, min_samples_leaf=1, n_estimators=10, total=   1.8s\n","[CV] max_depth=10, min_samples_leaf=1, n_estimators=100 ..............\n","[CV]  max_depth=10, min_samples_leaf=1, n_estimators=100, total=  11.9s\n","[CV] max_depth=10, min_samples_leaf=1, n_estimators=100 ..............\n","[CV]  max_depth=10, min_samples_leaf=1, n_estimators=100, total=  12.0s\n","[CV] max_depth=10, min_samples_leaf=1, n_estimators=100 ..............\n","[CV]  max_depth=10, min_samples_leaf=1, n_estimators=100, total=  11.9s\n","[CV] max_depth=10, min_samples_leaf=1, n_estimators=100 ..............\n","[CV]  max_depth=10, min_samples_leaf=1, n_estimators=100, total=  11.8s\n","[CV] max_depth=10, min_samples_leaf=1, n_estimators=100 ..............\n","[CV]  max_depth=10, min_samples_leaf=1, n_estimators=100, total=  11.8s\n","[CV] max_depth=10, min_samples_leaf=1, n_estimators=500 ..............\n","[CV]  max_depth=10, min_samples_leaf=1, n_estimators=500, total=  56.4s\n","[CV] max_depth=10, min_samples_leaf=1, n_estimators=500 ..............\n","[CV]  max_depth=10, min_samples_leaf=1, n_estimators=500, total=  56.7s\n","[CV] max_depth=10, min_samples_leaf=1, n_estimators=500 ..............\n","[CV]  max_depth=10, min_samples_leaf=1, n_estimators=500, total=  56.6s\n","[CV] max_depth=10, min_samples_leaf=1, n_estimators=500 ..............\n","[CV]  max_depth=10, min_samples_leaf=1, n_estimators=500, total=  56.5s\n","[CV] max_depth=10, min_samples_leaf=1, n_estimators=500 ..............\n","[CV]  max_depth=10, min_samples_leaf=1, n_estimators=500, total=  56.8s\n","[CV] max_depth=10, min_samples_leaf=1, n_estimators=1000 .............\n","[CV]  max_depth=10, min_samples_leaf=1, n_estimators=1000, total= 1.9min\n","[CV] max_depth=10, min_samples_leaf=1, n_estimators=1000 .............\n","[CV]  max_depth=10, min_samples_leaf=1, n_estimators=1000, total= 1.9min\n","[CV] max_depth=10, min_samples_leaf=1, n_estimators=1000 .............\n","[CV]  max_depth=10, min_samples_leaf=1, n_estimators=1000, total= 1.9min\n","[CV] max_depth=10, min_samples_leaf=1, n_estimators=1000 .............\n","[CV]  max_depth=10, min_samples_leaf=1, n_estimators=1000, total= 1.9min\n","[CV] max_depth=10, min_samples_leaf=1, n_estimators=1000 .............\n","[CV]  max_depth=10, min_samples_leaf=1, n_estimators=1000, total= 1.9min\n","[CV] max_depth=10, min_samples_leaf=2, n_estimators=10 ...............\n","[CV]  max_depth=10, min_samples_leaf=2, n_estimators=10, total=   1.8s\n","[CV] max_depth=10, min_samples_leaf=2, n_estimators=10 ...............\n","[CV]  max_depth=10, min_samples_leaf=2, n_estimators=10, total=   1.8s\n","[CV] max_depth=10, min_samples_leaf=2, n_estimators=10 ...............\n","[CV]  max_depth=10, min_samples_leaf=2, n_estimators=10, total=   1.8s\n","[CV] max_depth=10, min_samples_leaf=2, n_estimators=10 ...............\n","[CV]  max_depth=10, min_samples_leaf=2, n_estimators=10, total=   1.8s\n","[CV] max_depth=10, min_samples_leaf=2, n_estimators=10 ...............\n","[CV]  max_depth=10, min_samples_leaf=2, n_estimators=10, total=   1.8s\n","[CV] max_depth=10, min_samples_leaf=2, n_estimators=100 ..............\n","[CV]  max_depth=10, min_samples_leaf=2, n_estimators=100, total=  11.6s\n","[CV] max_depth=10, min_samples_leaf=2, n_estimators=100 ..............\n","[CV]  max_depth=10, min_samples_leaf=2, n_estimators=100, total=  11.6s\n","[CV] max_depth=10, min_samples_leaf=2, n_estimators=100 ..............\n","[CV]  max_depth=10, min_samples_leaf=2, n_estimators=100, total=  11.7s\n","[CV] max_depth=10, min_samples_leaf=2, n_estimators=100 ..............\n","[CV]  max_depth=10, min_samples_leaf=2, n_estimators=100, total=  11.7s\n","[CV] max_depth=10, min_samples_leaf=2, n_estimators=100 ..............\n","[CV]  max_depth=10, min_samples_leaf=2, n_estimators=100, total=  11.6s\n","[CV] max_depth=10, min_samples_leaf=2, n_estimators=500 ..............\n","[CV]  max_depth=10, min_samples_leaf=2, n_estimators=500, total=  54.9s\n","[CV] max_depth=10, min_samples_leaf=2, n_estimators=500 ..............\n","[CV]  max_depth=10, min_samples_leaf=2, n_estimators=500, total=  55.4s\n","[CV] max_depth=10, min_samples_leaf=2, n_estimators=500 ..............\n","[CV]  max_depth=10, min_samples_leaf=2, n_estimators=500, total=  55.0s\n","[CV] max_depth=10, min_samples_leaf=2, n_estimators=500 ..............\n","[CV]  max_depth=10, min_samples_leaf=2, n_estimators=500, total=  55.3s\n","[CV] max_depth=10, min_samples_leaf=2, n_estimators=500 ..............\n","[CV]  max_depth=10, min_samples_leaf=2, n_estimators=500, total=  55.1s\n","[CV] max_depth=10, min_samples_leaf=2, n_estimators=1000 .............\n","[CV]  max_depth=10, min_samples_leaf=2, n_estimators=1000, total= 1.8min\n","[CV] max_depth=10, min_samples_leaf=2, n_estimators=1000 .............\n","[CV]  max_depth=10, min_samples_leaf=2, n_estimators=1000, total= 2.3min\n","[CV] max_depth=10, min_samples_leaf=2, n_estimators=1000 .............\n","[CV]  max_depth=10, min_samples_leaf=2, n_estimators=1000, total= 1.8min\n","[CV] max_depth=10, min_samples_leaf=2, n_estimators=1000 .............\n","[CV]  max_depth=10, min_samples_leaf=2, n_estimators=1000, total= 1.8min\n","[CV] max_depth=10, min_samples_leaf=2, n_estimators=1000 .............\n","[CV]  max_depth=10, min_samples_leaf=2, n_estimators=1000, total= 1.8min\n","[CV] max_depth=10, min_samples_leaf=5, n_estimators=10 ...............\n","[CV]  max_depth=10, min_samples_leaf=5, n_estimators=10, total=   1.8s\n","[CV] max_depth=10, min_samples_leaf=5, n_estimators=10 ...............\n","[CV]  max_depth=10, min_samples_leaf=5, n_estimators=10, total=   1.8s\n","[CV] max_depth=10, min_samples_leaf=5, n_estimators=10 ...............\n","[CV]  max_depth=10, min_samples_leaf=5, n_estimators=10, total=   1.8s\n","[CV] max_depth=10, min_samples_leaf=5, n_estimators=10 ...............\n","[CV]  max_depth=10, min_samples_leaf=5, n_estimators=10, total=   1.8s\n","[CV] max_depth=10, min_samples_leaf=5, n_estimators=10 ...............\n","[CV]  max_depth=10, min_samples_leaf=5, n_estimators=10, total=   1.8s\n","[CV] max_depth=10, min_samples_leaf=5, n_estimators=100 ..............\n","[CV]  max_depth=10, min_samples_leaf=5, n_estimators=100, total=  11.2s\n","[CV] max_depth=10, min_samples_leaf=5, n_estimators=100 ..............\n","[CV]  max_depth=10, min_samples_leaf=5, n_estimators=100, total=  11.1s\n","[CV] max_depth=10, min_samples_leaf=5, n_estimators=100 ..............\n","[CV]  max_depth=10, min_samples_leaf=5, n_estimators=100, total=  11.2s\n","[CV] max_depth=10, min_samples_leaf=5, n_estimators=100 ..............\n","[CV]  max_depth=10, min_samples_leaf=5, n_estimators=100, total=  11.3s\n","[CV] max_depth=10, min_samples_leaf=5, n_estimators=100 ..............\n","[CV]  max_depth=10, min_samples_leaf=5, n_estimators=100, total=  11.2s\n","[CV] max_depth=10, min_samples_leaf=5, n_estimators=500 ..............\n","[CV]  max_depth=10, min_samples_leaf=5, n_estimators=500, total=  53.3s\n","[CV] max_depth=10, min_samples_leaf=5, n_estimators=500 ..............\n","[CV]  max_depth=10, min_samples_leaf=5, n_estimators=500, total=  52.9s\n","[CV] max_depth=10, min_samples_leaf=5, n_estimators=500 ..............\n","[CV]  max_depth=10, min_samples_leaf=5, n_estimators=500, total=  53.0s\n","[CV] max_depth=10, min_samples_leaf=5, n_estimators=500 ..............\n","[CV]  max_depth=10, min_samples_leaf=5, n_estimators=500, total=  52.9s\n","[CV] max_depth=10, min_samples_leaf=5, n_estimators=500 ..............\n","[CV]  max_depth=10, min_samples_leaf=5, n_estimators=500, total=  52.9s\n","[CV] max_depth=10, min_samples_leaf=5, n_estimators=1000 .............\n","[CV]  max_depth=10, min_samples_leaf=5, n_estimators=1000, total= 1.8min\n","[CV] max_depth=10, min_samples_leaf=5, n_estimators=1000 .............\n","[CV]  max_depth=10, min_samples_leaf=5, n_estimators=1000, total= 1.8min\n","[CV] max_depth=10, min_samples_leaf=5, n_estimators=1000 .............\n","[CV]  max_depth=10, min_samples_leaf=5, n_estimators=1000, total= 1.8min\n","[CV] max_depth=10, min_samples_leaf=5, n_estimators=1000 .............\n","[CV]  max_depth=10, min_samples_leaf=5, n_estimators=1000, total= 1.8min\n","[CV] max_depth=10, min_samples_leaf=5, n_estimators=1000 .............\n","[CV]  max_depth=10, min_samples_leaf=5, n_estimators=1000, total= 1.8min\n","[CV] max_depth=15, min_samples_leaf=1, n_estimators=10 ...............\n","[CV]  max_depth=15, min_samples_leaf=1, n_estimators=10, total=   2.3s\n","[CV] max_depth=15, min_samples_leaf=1, n_estimators=10 ...............\n","[CV]  max_depth=15, min_samples_leaf=1, n_estimators=10, total=   2.3s\n","[CV] max_depth=15, min_samples_leaf=1, n_estimators=10 ...............\n","[CV]  max_depth=15, min_samples_leaf=1, n_estimators=10, total=   2.3s\n","[CV] max_depth=15, min_samples_leaf=1, n_estimators=10 ...............\n","[CV]  max_depth=15, min_samples_leaf=1, n_estimators=10, total=   2.2s\n","[CV] max_depth=15, min_samples_leaf=1, n_estimators=10 ...............\n","[CV]  max_depth=15, min_samples_leaf=1, n_estimators=10, total=   2.3s\n","[CV] max_depth=15, min_samples_leaf=1, n_estimators=100 ..............\n","[CV]  max_depth=15, min_samples_leaf=1, n_estimators=100, total=  16.2s\n","[CV] max_depth=15, min_samples_leaf=1, n_estimators=100 ..............\n","[CV]  max_depth=15, min_samples_leaf=1, n_estimators=100, total=  16.2s\n","[CV] max_depth=15, min_samples_leaf=1, n_estimators=100 ..............\n","[CV]  max_depth=15, min_samples_leaf=1, n_estimators=100, total=  16.1s\n","[CV] max_depth=15, min_samples_leaf=1, n_estimators=100 ..............\n","[CV]  max_depth=15, min_samples_leaf=1, n_estimators=100, total=  16.1s\n","[CV] max_depth=15, min_samples_leaf=1, n_estimators=100 ..............\n","[CV]  max_depth=15, min_samples_leaf=1, n_estimators=100, total=  16.2s\n","[CV] max_depth=15, min_samples_leaf=1, n_estimators=500 ..............\n","[CV]  max_depth=15, min_samples_leaf=1, n_estimators=500, total= 1.3min\n","[CV] max_depth=15, min_samples_leaf=1, n_estimators=500 ..............\n","[CV]  max_depth=15, min_samples_leaf=1, n_estimators=500, total= 1.3min\n","[CV] max_depth=15, min_samples_leaf=1, n_estimators=500 ..............\n","[CV]  max_depth=15, min_samples_leaf=1, n_estimators=500, total= 1.3min\n","[CV] max_depth=15, min_samples_leaf=1, n_estimators=500 ..............\n","[CV]  max_depth=15, min_samples_leaf=1, n_estimators=500, total= 1.3min\n","[CV] max_depth=15, min_samples_leaf=1, n_estimators=500 ..............\n","[CV]  max_depth=15, min_samples_leaf=1, n_estimators=500, total= 1.3min\n","[CV] max_depth=15, min_samples_leaf=1, n_estimators=1000 .............\n","[CV]  max_depth=15, min_samples_leaf=1, n_estimators=1000, total= 2.6min\n","[CV] max_depth=15, min_samples_leaf=1, n_estimators=1000 .............\n","[CV]  max_depth=15, min_samples_leaf=1, n_estimators=1000, total= 2.6min\n","[CV] max_depth=15, min_samples_leaf=1, n_estimators=1000 .............\n","[CV]  max_depth=15, min_samples_leaf=1, n_estimators=1000, total= 2.6min\n","[CV] max_depth=15, min_samples_leaf=1, n_estimators=1000 .............\n","[CV]  max_depth=15, min_samples_leaf=1, n_estimators=1000, total= 2.6min\n","[CV] max_depth=15, min_samples_leaf=1, n_estimators=1000 .............\n","[CV]  max_depth=15, min_samples_leaf=1, n_estimators=1000, total= 2.6min\n","[CV] max_depth=15, min_samples_leaf=2, n_estimators=10 ...............\n","[CV]  max_depth=15, min_samples_leaf=2, n_estimators=10, total=   2.3s\n","[CV] max_depth=15, min_samples_leaf=2, n_estimators=10 ...............\n","[CV]  max_depth=15, min_samples_leaf=2, n_estimators=10, total=   2.2s\n","[CV] max_depth=15, min_samples_leaf=2, n_estimators=10 ...............\n","[CV]  max_depth=15, min_samples_leaf=2, n_estimators=10, total=   2.2s\n","[CV] max_depth=15, min_samples_leaf=2, n_estimators=10 ...............\n","[CV]  max_depth=15, min_samples_leaf=2, n_estimators=10, total=   2.2s\n","[CV] max_depth=15, min_samples_leaf=2, n_estimators=10 ...............\n","[CV]  max_depth=15, min_samples_leaf=2, n_estimators=10, total=   2.2s\n","[CV] max_depth=15, min_samples_leaf=2, n_estimators=100 ..............\n","[CV]  max_depth=15, min_samples_leaf=2, n_estimators=100, total=  15.5s\n","[CV] max_depth=15, min_samples_leaf=2, n_estimators=100 ..............\n","[CV]  max_depth=15, min_samples_leaf=2, n_estimators=100, total=  15.4s\n","[CV] max_depth=15, min_samples_leaf=2, n_estimators=100 ..............\n","[CV]  max_depth=15, min_samples_leaf=2, n_estimators=100, total=  15.4s\n","[CV] max_depth=15, min_samples_leaf=2, n_estimators=100 ..............\n","[CV]  max_depth=15, min_samples_leaf=2, n_estimators=100, total=  15.5s\n","[CV] max_depth=15, min_samples_leaf=2, n_estimators=100 ..............\n","[CV]  max_depth=15, min_samples_leaf=2, n_estimators=100, total=  15.6s\n","[CV] max_depth=15, min_samples_leaf=2, n_estimators=500 ..............\n","[CV]  max_depth=15, min_samples_leaf=2, n_estimators=500, total= 1.3min\n","[CV] max_depth=15, min_samples_leaf=2, n_estimators=500 ..............\n","[CV]  max_depth=15, min_samples_leaf=2, n_estimators=500, total= 1.3min\n","[CV] max_depth=15, min_samples_leaf=2, n_estimators=500 ..............\n","[CV]  max_depth=15, min_samples_leaf=2, n_estimators=500, total= 1.3min\n","[CV] max_depth=15, min_samples_leaf=2, n_estimators=500 ..............\n","[CV]  max_depth=15, min_samples_leaf=2, n_estimators=500, total= 1.3min\n","[CV] max_depth=15, min_samples_leaf=2, n_estimators=500 ..............\n","[CV]  max_depth=15, min_samples_leaf=2, n_estimators=500, total= 1.3min\n","[CV] max_depth=15, min_samples_leaf=2, n_estimators=1000 .............\n","[CV]  max_depth=15, min_samples_leaf=2, n_estimators=1000, total= 2.5min\n","[CV] max_depth=15, min_samples_leaf=2, n_estimators=1000 .............\n","[CV]  max_depth=15, min_samples_leaf=2, n_estimators=1000, total= 2.5min\n","[CV] max_depth=15, min_samples_leaf=2, n_estimators=1000 .............\n","[CV]  max_depth=15, min_samples_leaf=2, n_estimators=1000, total= 2.5min\n","[CV] max_depth=15, min_samples_leaf=2, n_estimators=1000 .............\n","[CV]  max_depth=15, min_samples_leaf=2, n_estimators=1000, total= 2.5min\n","[CV] max_depth=15, min_samples_leaf=2, n_estimators=1000 .............\n","[CV]  max_depth=15, min_samples_leaf=2, n_estimators=1000, total= 2.5min\n","[CV] max_depth=15, min_samples_leaf=5, n_estimators=10 ...............\n","[CV]  max_depth=15, min_samples_leaf=5, n_estimators=10, total=   2.1s\n","[CV] max_depth=15, min_samples_leaf=5, n_estimators=10 ...............\n","[CV]  max_depth=15, min_samples_leaf=5, n_estimators=10, total=   2.2s\n","[CV] max_depth=15, min_samples_leaf=5, n_estimators=10 ...............\n","[CV]  max_depth=15, min_samples_leaf=5, n_estimators=10, total=   2.2s\n","[CV] max_depth=15, min_samples_leaf=5, n_estimators=10 ...............\n","[CV]  max_depth=15, min_samples_leaf=5, n_estimators=10, total=   2.2s\n","[CV] max_depth=15, min_samples_leaf=5, n_estimators=10 ...............\n","[CV]  max_depth=15, min_samples_leaf=5, n_estimators=10, total=   2.2s\n","[CV] max_depth=15, min_samples_leaf=5, n_estimators=100 ..............\n","[CV]  max_depth=15, min_samples_leaf=5, n_estimators=100, total=  14.7s\n","[CV] max_depth=15, min_samples_leaf=5, n_estimators=100 ..............\n","[CV]  max_depth=15, min_samples_leaf=5, n_estimators=100, total=  18.1s\n","[CV] max_depth=15, min_samples_leaf=5, n_estimators=100 ..............\n","[CV]  max_depth=15, min_samples_leaf=5, n_estimators=100, total=  18.1s\n","[CV] max_depth=15, min_samples_leaf=5, n_estimators=100 ..............\n","[CV]  max_depth=15, min_samples_leaf=5, n_estimators=100, total=  18.1s\n","[CV] max_depth=15, min_samples_leaf=5, n_estimators=100 ..............\n","[CV]  max_depth=15, min_samples_leaf=5, n_estimators=100, total=  17.9s\n","[CV] max_depth=15, min_samples_leaf=5, n_estimators=500 ..............\n","[CV]  max_depth=15, min_samples_leaf=5, n_estimators=500, total= 1.2min\n","[CV] max_depth=15, min_samples_leaf=5, n_estimators=500 ..............\n","[CV]  max_depth=15, min_samples_leaf=5, n_estimators=500, total= 1.2min\n","[CV] max_depth=15, min_samples_leaf=5, n_estimators=500 ..............\n","[CV]  max_depth=15, min_samples_leaf=5, n_estimators=500, total= 1.2min\n","[CV] max_depth=15, min_samples_leaf=5, n_estimators=500 ..............\n","[CV]  max_depth=15, min_samples_leaf=5, n_estimators=500, total= 1.2min\n","[CV] max_depth=15, min_samples_leaf=5, n_estimators=500 ..............\n","[CV]  max_depth=15, min_samples_leaf=5, n_estimators=500, total= 1.2min\n","[CV] max_depth=15, min_samples_leaf=5, n_estimators=1000 .............\n","[CV]  max_depth=15, min_samples_leaf=5, n_estimators=1000, total= 2.3min\n","[CV] max_depth=15, min_samples_leaf=5, n_estimators=1000 .............\n","[CV]  max_depth=15, min_samples_leaf=5, n_estimators=1000, total= 2.9min\n","[CV] max_depth=15, min_samples_leaf=5, n_estimators=1000 .............\n","[CV]  max_depth=15, min_samples_leaf=5, n_estimators=1000, total= 2.3min\n","[CV] max_depth=15, min_samples_leaf=5, n_estimators=1000 .............\n","[CV]  max_depth=15, min_samples_leaf=5, n_estimators=1000, total= 2.3min\n","[CV] max_depth=15, min_samples_leaf=5, n_estimators=1000 .............\n","[CV]  max_depth=15, min_samples_leaf=5, n_estimators=1000, total= 2.3min\n"],"name":"stdout"},{"output_type":"stream","text":["[Parallel(n_jobs=1)]: Done 240 out of 240 | elapsed: 230.7min finished\n"],"name":"stderr"},{"output_type":"stream","text":["{'max_depth': 15, 'min_samples_leaf': 5, 'n_estimators': 1000}\n","RandomForestClassifier(bootstrap=True, ccp_alpha=0.0, class_weight=None,\n","                       criterion='entropy', max_depth=15, max_features='sqrt',\n","                       max_leaf_nodes=None, max_samples=None,\n","                       min_impurity_decrease=0.0, min_impurity_split=None,\n","                       min_samples_leaf=5, min_samples_split=2,\n","                       min_weight_fraction_leaf=0.0, n_estimators=1000,\n","                       n_jobs=None, oob_score=False, random_state=42, verbose=0,\n","                       warm_start=False)\n","{'mean_fit_time': array([  3.17725468,  29.76168466, 150.53063316, 269.6014957 ,\n","         2.35120544,  22.43549848, 118.49562855, 242.60547099,\n","         2.18398728,  20.68777142, 103.86691227, 196.11157722,\n","         0.84222093,   6.29203572,  30.64689741,  61.64968052,\n","         0.78786922,   6.25896435,  30.39445848,  60.5715065 ,\n","         0.73844991,   6.03005462,  29.45691643,  58.8666327 ,\n","         1.24958897,  11.22256265,  55.54687676, 111.47103462,\n","         1.22796001,  10.96224022,  54.12400365, 113.84112673,\n","         1.18417897,  10.51991835,  51.95353827, 104.08555198,\n","         1.68229451,  15.43779459,  76.35145802, 153.55067706,\n","         1.62306671,  14.81300173,  74.85611277, 148.4305068 ,\n","         1.55258112,  16.6869421 ,  69.22597618, 145.32355027]), 'std_fit_time': array([2.18749344e-01, 1.67642850e-01, 1.19277378e+00, 1.22978316e+01,\n","       5.38973348e-02, 2.38731789e-01, 4.84011627e+00, 1.32776997e+01,\n","       5.96746232e-02, 1.92558949e-01, 7.73855347e-01, 5.14158418e+00,\n","       7.75393128e-02, 2.55093025e-02, 9.76434643e-02, 3.44598738e-01,\n","       8.18344226e-03, 1.97730007e-02, 2.24940212e-01, 2.33674002e-01,\n","       4.39200815e-03, 2.23051591e-02, 1.74066020e-01, 3.12481982e-01,\n","       9.36205834e-03, 5.03175557e-02, 1.28344330e-01, 2.19487075e-01,\n","       9.28010547e-03, 3.93379348e-02, 1.76005577e-01, 1.02732416e+01,\n","       1.10057869e-02, 4.99005636e-02, 1.35273555e-01, 4.06440994e-01,\n","       1.22240419e-02, 5.97439056e-02, 1.29872968e-01, 1.08594545e+00,\n","       3.39851759e-02, 6.61170111e-02, 4.34697324e-01, 4.27657569e-01,\n","       1.87976821e-02, 1.33409713e+00, 1.46671174e-01, 1.30783504e+01]), 'mean_score_time': array([0.5730094 , 0.79201097, 1.87013264, 3.08485603, 0.58443384,\n","       0.76737537, 1.80599093, 3.29074974, 0.59011674, 0.73027334,\n","       1.43549418, 2.21284928, 0.66605926, 0.63545327, 0.92681966,\n","       1.30922599, 0.57114558, 0.64177423, 0.93230925, 1.29634433,\n","       0.56672387, 0.63375497, 0.92468939, 1.29840155, 0.5743073 ,\n","       0.65423355, 1.04170942, 1.58697062, 0.58007412, 0.66296682,\n","       1.03428526, 1.52503238, 0.57591071, 0.66083093, 1.03082399,\n","       1.54018273, 0.58466311, 0.69685698, 1.17888899, 2.04243517,\n","       0.57818084, 0.68223987, 1.22918749, 1.88151965, 0.60301218,\n","       0.68858447, 1.17578988, 1.77085724]), 'std_score_time': array([0.01669299, 0.0025252 , 0.0582238 , 0.14258599, 0.00729405,\n","       0.00719565, 0.18384489, 0.4515456 , 0.01056358, 0.00508877,\n","       0.01762973, 0.1884293 , 0.19794715, 0.00816857, 0.00914896,\n","       0.01537595, 0.00453138, 0.00793144, 0.00710276, 0.00879838,\n","       0.00572118, 0.00489291, 0.00346495, 0.01236161, 0.0031606 ,\n","       0.01187607, 0.00485841, 0.04677583, 0.00398872, 0.01051511,\n","       0.01036888, 0.01595569, 0.00364078, 0.00906763, 0.01388446,\n","       0.01916755, 0.00695502, 0.00571633, 0.01050294, 0.38047008,\n","       0.00986742, 0.00594   , 0.03052144, 0.02495269, 0.01225384,\n","       0.01100048, 0.03985895, 0.05650212]), 'param_max_depth': masked_array(data=[None, None, None, None, None, None, None, None, None,\n","                   None, None, None, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,\n","                   10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 15, 15,\n","                   15, 15, 15, 15, 15, 15, 15, 15, 15, 15],\n","             mask=[False, False, False, False, False, False, False, False,\n","                   False, False, False, False, False, False, False, False,\n","                   False, False, False, False, False, False, False, False,\n","                   False, False, False, False, False, False, False, False,\n","                   False, False, False, False, False, False, False, False,\n","                   False, False, False, False, False, False, False, False],\n","       fill_value='?',\n","            dtype=object), 'param_min_samples_leaf': masked_array(data=[1, 1, 1, 1, 2, 2, 2, 2, 5, 5, 5, 5, 1, 1, 1, 1, 2, 2,\n","                   2, 2, 5, 5, 5, 5, 1, 1, 1, 1, 2, 2, 2, 2, 5, 5, 5, 5,\n","                   1, 1, 1, 1, 2, 2, 2, 2, 5, 5, 5, 5],\n","             mask=[False, False, False, False, False, False, False, False,\n","                   False, False, False, False, False, False, False, False,\n","                   False, False, False, False, False, False, False, False,\n","                   False, False, False, False, False, False, False, False,\n","                   False, False, False, False, False, False, False, False,\n","                   False, False, False, False, False, False, False, False],\n","       fill_value='?',\n","            dtype=object), 'param_n_estimators': masked_array(data=[10, 100, 500, 1000, 10, 100, 500, 1000, 10, 100, 500,\n","                   1000, 10, 100, 500, 1000, 10, 100, 500, 1000, 10, 100,\n","                   500, 1000, 10, 100, 500, 1000, 10, 100, 500, 1000, 10,\n","                   100, 500, 1000, 10, 100, 500, 1000, 10, 100, 500, 1000,\n","                   10, 100, 500, 1000],\n","             mask=[False, False, False, False, False, False, False, False,\n","                   False, False, False, False, False, False, False, False,\n","                   False, False, False, False, False, False, False, False,\n","                   False, False, False, False, False, False, False, False,\n","                   False, False, False, False, False, False, False, False,\n","                   False, False, False, False, False, False, False, False],\n","       fill_value='?',\n","            dtype=object), 'params': [{'max_depth': None, 'min_samples_leaf': 1, 'n_estimators': 10}, {'max_depth': None, 'min_samples_leaf': 1, 'n_estimators': 100}, {'max_depth': None, 'min_samples_leaf': 1, 'n_estimators': 500}, {'max_depth': None, 'min_samples_leaf': 1, 'n_estimators': 1000}, {'max_depth': None, 'min_samples_leaf': 2, 'n_estimators': 10}, {'max_depth': None, 'min_samples_leaf': 2, 'n_estimators': 100}, {'max_depth': None, 'min_samples_leaf': 2, 'n_estimators': 500}, {'max_depth': None, 'min_samples_leaf': 2, 'n_estimators': 1000}, {'max_depth': None, 'min_samples_leaf': 5, 'n_estimators': 10}, {'max_depth': None, 'min_samples_leaf': 5, 'n_estimators': 100}, {'max_depth': None, 'min_samples_leaf': 5, 'n_estimators': 500}, {'max_depth': None, 'min_samples_leaf': 5, 'n_estimators': 1000}, {'max_depth': 5, 'min_samples_leaf': 1, 'n_estimators': 10}, {'max_depth': 5, 'min_samples_leaf': 1, 'n_estimators': 100}, {'max_depth': 5, 'min_samples_leaf': 1, 'n_estimators': 500}, {'max_depth': 5, 'min_samples_leaf': 1, 'n_estimators': 1000}, {'max_depth': 5, 'min_samples_leaf': 2, 'n_estimators': 10}, {'max_depth': 5, 'min_samples_leaf': 2, 'n_estimators': 100}, {'max_depth': 5, 'min_samples_leaf': 2, 'n_estimators': 500}, {'max_depth': 5, 'min_samples_leaf': 2, 'n_estimators': 1000}, {'max_depth': 5, 'min_samples_leaf': 5, 'n_estimators': 10}, {'max_depth': 5, 'min_samples_leaf': 5, 'n_estimators': 100}, {'max_depth': 5, 'min_samples_leaf': 5, 'n_estimators': 500}, {'max_depth': 5, 'min_samples_leaf': 5, 'n_estimators': 1000}, {'max_depth': 10, 'min_samples_leaf': 1, 'n_estimators': 10}, {'max_depth': 10, 'min_samples_leaf': 1, 'n_estimators': 100}, {'max_depth': 10, 'min_samples_leaf': 1, 'n_estimators': 500}, {'max_depth': 10, 'min_samples_leaf': 1, 'n_estimators': 1000}, {'max_depth': 10, 'min_samples_leaf': 2, 'n_estimators': 10}, {'max_depth': 10, 'min_samples_leaf': 2, 'n_estimators': 100}, {'max_depth': 10, 'min_samples_leaf': 2, 'n_estimators': 500}, {'max_depth': 10, 'min_samples_leaf': 2, 'n_estimators': 1000}, {'max_depth': 10, 'min_samples_leaf': 5, 'n_estimators': 10}, {'max_depth': 10, 'min_samples_leaf': 5, 'n_estimators': 100}, {'max_depth': 10, 'min_samples_leaf': 5, 'n_estimators': 500}, {'max_depth': 10, 'min_samples_leaf': 5, 'n_estimators': 1000}, {'max_depth': 15, 'min_samples_leaf': 1, 'n_estimators': 10}, {'max_depth': 15, 'min_samples_leaf': 1, 'n_estimators': 100}, {'max_depth': 15, 'min_samples_leaf': 1, 'n_estimators': 500}, {'max_depth': 15, 'min_samples_leaf': 1, 'n_estimators': 1000}, {'max_depth': 15, 'min_samples_leaf': 2, 'n_estimators': 10}, {'max_depth': 15, 'min_samples_leaf': 2, 'n_estimators': 100}, {'max_depth': 15, 'min_samples_leaf': 2, 'n_estimators': 500}, {'max_depth': 15, 'min_samples_leaf': 2, 'n_estimators': 1000}, {'max_depth': 15, 'min_samples_leaf': 5, 'n_estimators': 10}, {'max_depth': 15, 'min_samples_leaf': 5, 'n_estimators': 100}, {'max_depth': 15, 'min_samples_leaf': 5, 'n_estimators': 500}, {'max_depth': 15, 'min_samples_leaf': 5, 'n_estimators': 1000}], 'split0_test_score': array([0.63692229, 0.80507008, 0.85594383, 0.86809922, 0.74560456,\n","       0.85400304, 0.87966908, 0.88253343, 0.79700262, 0.87211739,\n","       0.88344743, 0.88526333, 0.74863136, 0.86754212, 0.8829374 ,\n","       0.88532482, 0.75852103, 0.86769642, 0.88380117, 0.88609299,\n","       0.75597927, 0.86745201, 0.88330553, 0.88562183, 0.7878858 ,\n","       0.86698536, 0.88302059, 0.88768763, 0.77368701, 0.86913371,\n","       0.8870135 , 0.88886681, 0.78555343, 0.86642885, 0.88505592,\n","       0.88741491, 0.77410281, 0.86515371, 0.88645423, 0.88989035,\n","       0.78140298, 0.86631145, 0.88749098, 0.89051018, 0.78099207,\n","       0.86902416, 0.88756268, 0.88837072]), 'split1_test_score': array([0.62924206, 0.79528352, 0.85629885, 0.86521609, 0.7196271 ,\n","       0.85093986, 0.87744297, 0.88279388, 0.76020653, 0.86706406,\n","       0.8846705 , 0.88675203, 0.74983431, 0.86164268, 0.87984003,\n","       0.88105193, 0.7484155 , 0.86311162, 0.87896427, 0.88023682,\n","       0.74912825, 0.86354241, 0.88125827, 0.88135851, 0.7711952 ,\n","       0.86595351, 0.88518918, 0.88814091, 0.76835299, 0.86710553,\n","       0.88316102, 0.88757109, 0.78106715, 0.86637982, 0.88343736,\n","       0.88637185, 0.74933487, 0.85539501, 0.87893975, 0.88464903,\n","       0.75963613, 0.86591306, 0.88084871, 0.88485805, 0.77680866,\n","       0.8696094 , 0.88679585, 0.88770732]), 'split2_test_score': array([0.62353332, 0.78825233, 0.84838464, 0.85777075, 0.74213435,\n","       0.8404162 , 0.86341476, 0.86807917, 0.77909817, 0.85183441,\n","       0.86766869, 0.87070481, 0.74816668, 0.85231742, 0.86536311,\n","       0.86810063, 0.75504621, 0.85147966, 0.86500138, 0.86914882,\n","       0.74838637, 0.85017576, 0.86610712, 0.86850042, 0.78636877,\n","       0.85337981, 0.86674966, 0.86945164, 0.76937878, 0.85650439,\n","       0.87092347, 0.87388188, 0.78567308, 0.85595168, 0.86984744,\n","       0.8718731 , 0.7644877 , 0.84585645, 0.86724215, 0.87217271,\n","       0.77088063, 0.85363425, 0.86964451, 0.87419127, 0.77138135,\n","       0.86139683, 0.87372634, 0.87395757]), 'split3_test_score': array([0.63986619, 0.80084073, 0.85205255, 0.8630986 , 0.73804586,\n","       0.84756198, 0.87927032, 0.88080713, 0.78077712, 0.86236221,\n","       0.87898758, 0.88134029, 0.76154127, 0.85518101, 0.8696383 ,\n","       0.8726212 , 0.76038779, 0.85480559, 0.87095609, 0.87347315,\n","       0.76511045, 0.85377305, 0.87012663, 0.87310157, 0.76917766,\n","       0.85426942, 0.87326245, 0.87693409, 0.77256073, 0.85564348,\n","       0.87722541, 0.87923001, 0.78102664, 0.86075264, 0.87452121,\n","       0.87657303, 0.77160142, 0.84584463, 0.87533831, 0.88006799,\n","       0.76729737, 0.85775983, 0.87896621, 0.88138565, 0.7819592 ,\n","       0.86187415, 0.87867413, 0.88051164]), 'split4_test_score': array([0.6294641 , 0.80239238, 0.84748646, 0.85811687, 0.73305909,\n","       0.84931859, 0.86950278, 0.87054271, 0.76669107, 0.85174162,\n","       0.86956314, 0.87132808, 0.74508215, 0.85680672, 0.87077392,\n","       0.87301333, 0.73980647, 0.85753451, 0.86929898, 0.87185429,\n","       0.74905578, 0.86094274, 0.87190928, 0.87251925, 0.78661087,\n","       0.86660618, 0.8714902 , 0.87301215, 0.78563564, 0.8631723 ,\n","       0.871275  , 0.87413609, 0.77860388, 0.8681025 , 0.87305776,\n","       0.8749154 , 0.76005335, 0.85056015, 0.86987145, 0.87298393,\n","       0.75924758, 0.85718445, 0.87012179, 0.87434556, 0.76842715,\n","       0.87032087, 0.87693695, 0.87760834]), 'mean_test_score': array([0.63180559, 0.79836781, 0.85203327, 0.86246031, 0.73569419,\n","       0.84844793, 0.87385998, 0.87695126, 0.7767551 , 0.86102394,\n","       0.87686747, 0.87907771, 0.75065115, 0.85869799, 0.87371055,\n","       0.87602238, 0.7524354 , 0.85892556, 0.87360438, 0.87616121,\n","       0.75353202, 0.85917719, 0.87454136, 0.87622031, 0.78024766,\n","       0.86143885, 0.87594242, 0.87904528, 0.77392303, 0.86231188,\n","       0.87791968, 0.88073718, 0.78238484, 0.8635231 , 0.87718394,\n","       0.87942966, 0.76391603, 0.85256199, 0.87556918, 0.8799528 ,\n","       0.76769294, 0.86016061, 0.87741444, 0.88105814, 0.77591369,\n","       0.86644508, 0.88073919, 0.88163112]), 'std_test_score': array([0.00585899, 0.00598446, 0.00367353, 0.00401638, 0.00905888,\n","       0.00454138, 0.0063866 , 0.00632373, 0.01270251, 0.00814798,\n","       0.00702351, 0.00681819, 0.00566618, 0.00535813, 0.00659685,\n","       0.00624833, 0.00751986, 0.00581036, 0.00681564, 0.00616765,\n","       0.00641805, 0.00633838, 0.00662543, 0.00628985, 0.00825581,\n","       0.00623211, 0.00703014, 0.0076199 , 0.00617693, 0.00544887,\n","       0.00638357, 0.00641302, 0.00278313, 0.00452939, 0.00598353,\n","       0.00628648, 0.00883961, 0.00722041, 0.00680738, 0.00678112,\n","       0.00817948, 0.00506237, 0.00677191, 0.00626259, 0.00528639,\n","       0.00395132, 0.00549808, 0.00563315]), 'rank_test_score': array([48, 36, 34, 25, 47, 35, 20, 12, 39, 28, 13,  7, 46, 32, 21, 16, 45,\n","       31, 22, 15, 44, 30, 19, 14, 38, 27, 17,  8, 41, 26,  9,  4, 37, 24,\n","       11,  6, 43, 33, 18,  5, 42, 29, 10,  2, 40, 23,  3,  1],\n","      dtype=int32)}\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"yhbb1HnO5xvC","executionInfo":{"status":"ok","timestamp":1620878769266,"user_tz":360,"elapsed":466,"user":{"displayName":"Laura Elliott","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiR_lgn2QjRHbMFy-LVQ-MZiwLY0GVJVCQPQcKj=s64","userId":"17671003486597139296"}},"outputId":"03701a5b-9d81-4732-cc00-5250fd41d180"},"source":["'''\n","4.2 RandomForestClassifier(bootstrap=True, ccp_alpha=0.0, class_weight=None,\n","                       criterion='entropy', max_depth=15, max_features='sqrt',\n","                       max_leaf_nodes=None, max_samples=None,\n","                       min_impurity_decrease=0.0, min_impurity_split=None,\n","                       min_samples_leaf=5, min_samples_split=2,\n","                       min_weight_fraction_leaf=0.0, n_estimators=1000,\n","                       n_jobs=None, oob_score=False, random_state=42, verbose=0,\n","                       warm_start=False)\n","'''\n","print(best_grid)\n"],"execution_count":null,"outputs":[{"output_type":"stream","text":["RandomForestClassifier(bootstrap=True, ccp_alpha=0.0, class_weight=None,\n","                       criterion='entropy', max_depth=None, max_features='sqrt',\n","                       max_leaf_nodes=None, max_samples=None,\n","                       min_impurity_decrease=0.0, min_impurity_split=None,\n","                       min_samples_leaf=1, min_samples_split=2,\n","                       min_weight_fraction_leaf=0.0, n_estimators=100,\n","                       n_jobs=None, oob_score=False, random_state=42, verbose=0,\n","                       warm_start=False)\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"yr6SdykH6ISw","executionInfo":{"status":"ok","timestamp":1620878863547,"user_tz":360,"elapsed":453,"user":{"displayName":"Laura Elliott","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiR_lgn2QjRHbMFy-LVQ-MZiwLY0GVJVCQPQcKj=s64","userId":"17671003486597139296"}},"outputId":"b1d9f47d-6995-43f9-ded5-f1401209e8ab"},"source":["# 4.2 \n","print(grid_search.best_params_)\n","{'max_depth': 15, 'min_samples_leaf': 5, 'n_estimators': 1000}"],"execution_count":null,"outputs":[{"output_type":"stream","text":["{'n_estimators': 100}\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"zI1uUe0G7_k-","executionInfo":{"status":"ok","timestamp":1620879357728,"user_tz":360,"elapsed":1011,"user":{"displayName":"Laura Elliott","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiR_lgn2QjRHbMFy-LVQ-MZiwLY0GVJVCQPQcKj=s64","userId":"17671003486597139296"}},"outputId":"d7fc6c46-68f7-4e79-c56f-5afdeb26b70e"},"source":["print(grid_search.cv_results_)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["{'mean_fit_time': array([ 27.2721221 , 148.15297918, 297.07272348, 545.12819142]), 'std_fit_time': array([ 0.9856402 ,  5.57172965, 12.11067414,  3.14160984]), 'mean_score_time': array([0.24836831, 1.26323528, 2.56578836, 5.39648328]), 'std_score_time': array([0.00745542, 0.03476885, 0.10739159, 0.87424313]), 'param_n_estimators': masked_array(data=[100, 500, 1000, 2000],\n","             mask=[False, False, False, False],\n","       fill_value='?',\n","            dtype=object), 'params': [{'n_estimators': 100}, {'n_estimators': 500}, {'n_estimators': 1000}, {'n_estimators': 2000}], 'split0_test_score': array([0.34884696, 0.34842767, 0.34884696, 0.34884696]), 'split1_test_score': array([0.35025168, 0.34479866, 0.34437919, 0.34521812]), 'split2_test_score': array([0.34689597, 0.35067114, 0.35151007, 0.34857383]), 'split3_test_score': array([0.34815436, 0.34983221, 0.34815436, 0.34815436]), 'split4_test_score': array([0.34647651, 0.33766779, 0.34018456, 0.34060403]), 'mean_test_score': array([0.3481251 , 0.34627949, 0.34661503, 0.34627946]), 'std_test_score': array([0.00136183, 0.00475137, 0.00394112, 0.00312117]), 'rank_test_score': array([1, 3, 2, 4], dtype=int32)}\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"__puKZiuxyun","executionInfo":{"status":"ok","timestamp":1620944158678,"user_tz":360,"elapsed":186470,"user":{"displayName":"Laura Elliott","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiR_lgn2QjRHbMFy-LVQ-MZiwLY0GVJVCQPQcKj=s64","userId":"17671003486597139296"}},"outputId":"8408400c-05db-4ec3-92e4-47b815533106"},"source":["'''\n","4.3 Running optimized random forest model with best results from gridsearchcv\n","'''\n","rf_fnl_tfidf  = RandomForestClassifier(n_estimators=1000, max_features='sqrt', criterion = 'entropy', max_depth= 15, min_samples_leaf = 5,random_state = 42)\n","\n","rf_fnl_tfidf.fit(X_dev_tfidf_df,y_dev)"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["RandomForestClassifier(bootstrap=True, ccp_alpha=0.0, class_weight=None,\n","                       criterion='entropy', max_depth=15, max_features='sqrt',\n","                       max_leaf_nodes=None, max_samples=None,\n","                       min_impurity_decrease=0.0, min_impurity_split=None,\n","                       min_samples_leaf=5, min_samples_split=2,\n","                       min_weight_fraction_leaf=0.0, n_estimators=1000,\n","                       n_jobs=None, oob_score=False, random_state=42, verbose=0,\n","                       warm_start=False)"]},"metadata":{"tags":[]},"execution_count":12}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"vK7JFNiUy1LJ","executionInfo":{"status":"ok","timestamp":1620944176220,"user_tz":360,"elapsed":10447,"user":{"displayName":"Laura Elliott","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiR_lgn2QjRHbMFy-LVQ-MZiwLY0GVJVCQPQcKj=s64","userId":"17671003486597139296"}},"outputId":"125d5521-ac8b-47cf-f632-0efd5adb89b4"},"source":["'''\n","4.4 Print scores - AUC score for both dev and cv sets; do optimized rf params reduce overfitting - yes but still pretty highly overfitted\n","'''\n","y_tfidf_prob_dev_fnl = rf_fnl_tfidf.predict_proba(X_dev_tfidf_df)\n","roc_auc_tfidf_dev_fnl = roc_auc_score(y_dev, y_tfidf_prob_dev_fnl, multi_class=\"ovo\",\n","                                  average=\"macro\")\n","\n","print(\"TFIDF RF Train AUC Score:\", round(roc_auc_tfidf_dev_fnl,4))\n","\n","y_tfidf_prob_val_fnl = rf_fnl_tfidf.predict_proba(X_cv_tfidf)\n","roc_auc_tfidf_val_fnl = roc_auc_score(y_cv, y_tfidf_prob_val_fnl, multi_class=\"ovo\",\n","                                  average=\"macro\")\n","print(\"TFIDF RF Val AUC Score:   \", round(roc_auc_tfidf_val_fnl,4))"],"execution_count":null,"outputs":[{"output_type":"stream","text":["TFIDF RF Train AUC Score: 0.976\n","TFIDF RF Val AUC Score:    0.8836\n"],"name":"stdout"}]}]}