behavioral-scorecard-for-internal-liability-custom (1).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/kaggle/input/hdfc-hackerearth-dataset/Test.csv\n",
      "/kaggle/input/hdfc-hackerearth-dataset/Train.csv\n"
     ]
    }
   ],
   "source": [
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load in \n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "# Input data files are available in the \"../input/\" directory.\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n",
    "\n",
    "import os\n",
    "for dirname, _, filenames in os.walk('/kaggle/input'):\n",
    "    for filename in filenames:\n",
    "        print(os.path.join(dirname, filename))\n",
    "        \n",
    "\n",
    "# Any results you write to the current directory are saved as output."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [],
   "source": [
    "\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import accuracy_score  \n",
    "import matplotlib.pyplot as plt\n",
    "import sklearn.metrics as metrics\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.ensemble import RandomForestClassifier\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv('/kaggle/input/hdfc-hackerearth-dataset/Train.csv')\n",
    "test = pd.read_csv('/kaggle/input/hdfc-hackerearth-dataset/Test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(17521, 2395)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#cm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Duplicate the data for class 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "concat_df =  train[train['Col2']==1].sample(700)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.concat(objs=[train, concat_df], axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "15760\n",
      "2461\n",
      "0.8649360627846989\n",
      "0.13506393721530102\n"
     ]
    }
   ],
   "source": [
    "class_0 =len(train[train['Col2']==0])\n",
    "class_1 =len(train[train['Col2']==1])\n",
    "\n",
    "print(class_0)\n",
    "print(class_1)\n",
    "print(class_0/(class_0+class_1))\n",
    "print(class_1/(class_0+class_1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "target = train['Col2']\n",
    "features  = train.columns.tolist()\n",
    "features.remove('Col1')\n",
    "features.remove('Col2')\n",
    "train_input  = train[features] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_null_column(df,percentage = 60):\n",
    "    rows = df.shape[0]\n",
    "    null_percantage_columns = (df.isna().sum()/rows)*100\n",
    "    null_percantage_columns = pd.DataFrame(null_percantage_columns).reset_index()\n",
    "    null_percantage_columns.columns = ['columns','percentage_null']\n",
    "    columns_removed = null_percantage_columns[null_percantage_columns['percentage_null']>=percentage]\n",
    "    null_percantage_columns = null_percantage_columns[null_percantage_columns['percentage_null']<percentage]\n",
    "    \n",
    "    df = df[null_percantage_columns['columns'].tolist()]\n",
    "    return df ,columns_removed\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_null_removed,columns_removed = remove_null_column(train_input,50)\n",
    "test_null_removed = test[train_null_removed.columns.tolist()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def thershold_matrix(model,x_text,Y_test):\t\n",
    "\tpred_proba_df = pd.DataFrame(model.predict_proba(x_test))\n",
    "\tthreshold_list = [0.05,0.1,0.15,0.2,0.25,0.3,0.35,0.4,0.45,0.5,0.55,0.6,0.65,.7,.75,.8,.85,.9,.95,.99]\n",
    "\tcolumns = ['Threshold', 'Accuracy', 'Precision','Recall','F1','TN','FP','FN','TP']\n",
    "\tscore_df=pd.DataFrame(columns=columns) \n",
    "\tfor i in threshold_list:\n",
    "\t\tY_test_pred = pred_proba_df.applymap(lambda x: 1 if x>i else 0)\n",
    "\t\ttest_accuracy = metrics.accuracy_score(Y_test.as_matrix().reshape(Y_test.as_matrix().size,1),\n",
    "\t\t\t\t\t\t\t\t\t\t\tY_test_pred.iloc[:,1].as_matrix().reshape(Y_test_pred.iloc[:,1].as_matrix().size,1))\n",
    "\t\ttest_precision = metrics.precision_score(Y_test.as_matrix().reshape(Y_test.as_matrix().size,1), Y_test_pred.iloc[:,1].as_matrix().reshape(Y_test_pred.iloc[:,1].as_matrix().size,1))\n",
    "\t\ttest_recall = metrics.recall_score(Y_test.as_matrix().reshape(Y_test.as_matrix().size,1), Y_test_pred.iloc[:,1].as_matrix().reshape(Y_test_pred.iloc[:,1].as_matrix().size,1))\n",
    "\t\ttest_f1 = metrics.f1_score(Y_test.as_matrix().reshape(Y_test.as_matrix().size,1), Y_test_pred.iloc[:,1].as_matrix().reshape(Y_test_pred.iloc[:,1].as_matrix().size,1),average = 'weighted')\n",
    "\t\tTN, FP, FN, TP = confusion_matrix(Y_test.as_matrix().reshape(Y_test.as_matrix().size,1), Y_test_pred.iloc[:,1].as_matrix().reshape(Y_test_pred.iloc[:,1].as_matrix().size,1)).ravel()\n",
    "\t\tscore_df.loc[len(score_df)]=[i,test_accuracy,test_precision,test_recall,test_f1,TN, FP, FN, TP]\n",
    "\treturn score_df\n",
    "\t\t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#dataset = [train_null_removed,test_null_removed]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "cnt = 0\n",
    "cat_column = []\n",
    "cont_column = []\n",
    "uniques = dict()\n",
    "for column in train_null_removed.columns:\n",
    "    unique = len(train_null_removed[column].unique())/int(train_null_removed.shape[0])\n",
    "    unique = unique*100\n",
    "    uniques[column] = unique\n",
    "    if unique < 0.05:\n",
    "        train_null_removed[column].fillna(train_null_removed[column].mode()[0], inplace = True)\n",
    "        test_null_removed[column].fillna(test_null_removed[column].mode()[0], inplace = True)\n",
    "        cnt = cnt+1\n",
    "        cat_column.append(column)\n",
    "    else:\n",
    "        train_null_removed[column].fillna(train_null_removed[column].mean(), inplace = True)\n",
    "        test_null_removed[column].fillna(test_null_removed[column].mean(), inplace = True)\n",
    "        cont_column.append(column)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1543\n",
      "628\n"
     ]
    }
   ],
   "source": [
    "print(len(cat_column))\n",
    "print(len(cont_column))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "x_train,x_test,y_train,y_test = train_test_split(train_null_removed,target,test_size = .25, random_state = 24,stratify=target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "lg_fs = LogisticRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "                   intercept_scaling=1, l1_ratio=None, max_iter=100,\n",
       "                   multi_class='warn', n_jobs=None, penalty='l2',\n",
       "                   random_state=None, solver='warn', tol=0.0001, verbose=0,\n",
       "                   warm_start=False)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lg_fs.fit(x_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8612818261633012"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lg_fs.score(x_test,y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "logistic_score_df = thershold_matrix(lg_fs,x_test,y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Threshold</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Recall</th>\n",
       "      <th>F1</th>\n",
       "      <th>TN</th>\n",
       "      <th>FP</th>\n",
       "      <th>FN</th>\n",
       "      <th>TP</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.861282</td>\n",
       "      <td>0.345455</td>\n",
       "      <td>0.030894</td>\n",
       "      <td>0.807911</td>\n",
       "      <td>3905.0</td>\n",
       "      <td>36.0</td>\n",
       "      <td>596.0</td>\n",
       "      <td>19.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Threshold  Accuracy  Precision    Recall        F1      TN    FP     FN  \\\n",
       "9        0.5  0.861282   0.345455  0.030894  0.807911  3905.0  36.0  596.0   \n",
       "\n",
       "     TP  \n",
       "9  19.0  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logistic_score_df[logistic_score_df['F1']==logistic_score_df.F1.max()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = lg_fs.predict(test_null_removed)\n",
    "X = test.Col1\n",
    "df = pd.DataFrame({'Col1': X,'Col2': y_pred})\n",
    "df.to_csv('Linear_regress_Submission.csv',index= False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=10)]: Using backend ThreadingBackend with 10 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "building tree 1 of 1000\n",
      "building tree 2 of 1000\n",
      "building tree 3 of 1000\n",
      "building tree 4 of 1000\n",
      "building tree 5 of 1000\n",
      "building tree 6 of 1000\n",
      "building tree 7 of 1000\n",
      "building tree 8 of 1000\n",
      "building tree 9 of 1000\n",
      "building tree 10 of 1000\n",
      "building tree 11 of 1000\n",
      "building tree 12 of 1000\n",
      "building tree 13 of 1000\n",
      "building tree 14 of 1000\n",
      "building tree 15 of 1000\n",
      "building tree 16 of 1000\n",
      "building tree 17 of 1000\n",
      "building tree 18 of 1000\n",
      "building tree 19 of 1000\n",
      "building tree 20 of 1000\n",
      "building tree 21 of 1000\n",
      "building tree 22 of 1000\n",
      "building tree 23 of 1000\n",
      "building tree 24 of 1000\n",
      "building tree 25 of 1000\n",
      "building tree 26 of 1000\n",
      "building tree 27 of 1000\n",
      "building tree 28 of 1000\n",
      "building tree 29 of 1000\n",
      "building tree 30 of 1000\n",
      "building tree 31 of 1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=10)]: Done  21 tasks      | elapsed:    3.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "building tree 32 of 1000\n",
      "building tree 33 of 1000\n",
      "building tree 34 of 1000\n",
      "building tree 35 of 1000\n",
      "building tree 36 of 1000\n",
      "building tree 37 of 1000\n",
      "building tree 38 of 1000\n",
      "building tree 39 of 1000\n",
      "building tree 40 of 1000\n",
      "building tree 41 of 1000\n",
      "building tree 42 of 1000\n",
      "building tree 43 of 1000\n",
      "building tree 44 of 1000\n",
      "building tree 45 of 1000\n",
      "building tree 46 of 1000\n",
      "building tree 47 of 1000\n",
      "building tree 48 of 1000\n",
      "building tree 49 of 1000\n",
      "building tree 50 of 1000\n",
      "building tree 51 of 1000\n",
      "building tree 52 of 1000\n",
      "building tree 53 of 1000\n",
      "building tree 54 of 1000\n",
      "building tree 55 of 1000\n",
      "building tree 56 of 1000\n",
      "building tree 57 of 1000\n",
      "building tree 58 of 1000\n",
      "building tree 59 of 1000\n",
      "building tree 60 of 1000\n",
      "building tree 61 of 1000\n",
      "building tree 62 of 1000\n",
      "building tree 63 of 1000\n",
      "building tree 64 of 1000\n",
      "building tree 65 of 1000\n",
      "building tree 66 of 1000\n",
      "building tree 67 of 1000\n",
      "building tree 68 of 1000\n",
      "building tree 69 of 1000\n",
      "building tree 70 of 1000\n",
      "building tree 71 of 1000\n",
      "building tree 72 of 1000\n",
      "building tree 73 of 1000\n",
      "building tree 74 of 1000\n",
      "building tree 75 of 1000\n",
      "building tree 76 of 1000\n",
      "building tree 77 of 1000\n",
      "building tree 78 of 1000\n",
      "building tree 79 of 1000\n",
      "building tree 80 of 1000\n",
      "building tree 81 of 1000\n",
      "building tree 82 of 1000\n",
      "building tree 83 of 1000\n",
      "building tree 84 of 1000\n",
      "building tree 85 of 1000\n",
      "building tree 86 of 1000\n",
      "building tree 87 of 1000\n",
      "building tree 88 of 1000\n",
      "building tree 89 of 1000\n",
      "building tree 90 of 1000\n",
      "building tree 91 of 1000\n",
      "building tree 92 of 1000\n",
      "building tree 93 of 1000\n",
      "building tree 94 of 1000\n",
      "building tree 95 of 1000\n",
      "building tree 96 of 1000\n",
      "building tree 97 of 1000\n",
      "building tree 98 of 1000\n",
      "building tree 99 of 1000\n",
      "building tree 100 of 1000\n",
      "building tree 101 of 1000\n",
      "building tree 102 of 1000\n",
      "building tree 103 of 1000\n",
      "building tree 104 of 1000\n",
      "building tree 105 of 1000\n",
      "building tree 106 of 1000\n",
      "building tree 107 of 1000\n",
      "building tree 108 of 1000\n",
      "building tree 109 of 1000\n",
      "building tree 110 of 1000\n",
      "building tree 111 of 1000\n",
      "building tree 112 of 1000\n",
      "building tree 113 of 1000\n",
      "building tree 114 of 1000\n",
      "building tree 115 of 1000\n",
      "building tree 116 of 1000\n",
      "building tree 117 of 1000\n",
      "building tree 118 of 1000\n",
      "building tree 119 of 1000\n",
      "building tree 120 of 1000\n",
      "building tree 121 of 1000\n",
      "building tree 122 of 1000\n",
      "building tree 123 of 1000\n",
      "building tree 124 of 1000\n",
      "building tree 125 of 1000\n",
      "building tree 126 of 1000\n",
      "building tree 127 of 1000\n",
      "building tree 128 of 1000\n",
      "building tree 129 of 1000building tree 130 of 1000\n",
      "\n",
      "building tree 131 of 1000\n",
      "building tree 132 of 1000\n",
      "building tree 133 of 1000\n",
      "building tree 134 of 1000\n",
      "building tree 135 of 1000\n",
      "building tree 136 of 1000\n",
      "building tree 137 of 1000\n",
      "building tree 138 of 1000\n",
      "building tree 139 of 1000building tree 140 of 1000\n",
      "\n",
      "building tree 141 of 1000\n",
      "building tree 142 of 1000\n",
      "building tree 143 of 1000\n",
      "building tree 144 of 1000\n",
      "building tree 145 of 1000\n",
      "building tree 146 of 1000\n",
      "building tree 147 of 1000\n",
      "building tree 148 of 1000\n",
      "building tree 149 of 1000\n",
      "building tree 150 of 1000\n",
      "building tree 151 of 1000\n",
      "building tree 152 of 1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=10)]: Done 142 tasks      | elapsed:   17.4s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "building tree 153 of 1000\n",
      "building tree 154 of 1000\n",
      "building tree 155 of 1000\n",
      "building tree 156 of 1000\n",
      "building tree 157 of 1000\n",
      "building tree 158 of 1000\n",
      "building tree 159 of 1000\n",
      "building tree 160 of 1000\n",
      "building tree 161 of 1000\n",
      "building tree 162 of 1000\n",
      "building tree 163 of 1000\n",
      "building tree 164 of 1000\n",
      "building tree 165 of 1000\n",
      "building tree 166 of 1000\n",
      "building tree 167 of 1000\n",
      "building tree 168 of 1000\n",
      "building tree 169 of 1000\n",
      "building tree 170 of 1000\n",
      "building tree 171 of 1000building tree 172 of 1000\n",
      "\n",
      "building tree 173 of 1000\n",
      "building tree 174 of 1000\n",
      "building tree 175 of 1000\n",
      "building tree 176 of 1000\n",
      "building tree 177 of 1000\n",
      "building tree 178 of 1000\n",
      "building tree 179 of 1000\n",
      "building tree 180 of 1000\n",
      "building tree 181 of 1000\n",
      "building tree 182 of 1000\n",
      "building tree 183 of 1000\n",
      "building tree 184 of 1000\n",
      "building tree 185 of 1000\n",
      "building tree 186 of 1000\n",
      "building tree 187 of 1000\n",
      "building tree 188 of 1000\n",
      "building tree 189 of 1000\n",
      "building tree 190 of 1000\n",
      "building tree 191 of 1000\n",
      "building tree 192 of 1000\n",
      "building tree 193 of 1000\n",
      "building tree 194 of 1000\n",
      "building tree 195 of 1000\n",
      "building tree 196 of 1000\n",
      "building tree 197 of 1000\n",
      "building tree 198 of 1000\n",
      "building tree 199 of 1000building tree 200 of 1000\n",
      "\n",
      "building tree 201 of 1000building tree 202 of 1000\n",
      "\n",
      "building tree 203 of 1000\n",
      "building tree 204 of 1000\n",
      "building tree 205 of 1000\n",
      "building tree 206 of 1000\n",
      "building tree 207 of 1000\n",
      "building tree 208 of 1000\n",
      "building tree 209 of 1000\n",
      "building tree 210 of 1000\n",
      "building tree 211 of 1000\n",
      "building tree 212 of 1000\n",
      "building tree 213 of 1000\n",
      "building tree 214 of 1000\n",
      "building tree 215 of 1000\n",
      "building tree 216 of 1000\n",
      "building tree 217 of 1000\n",
      "building tree 218 of 1000\n",
      "building tree 219 of 1000\n",
      "building tree 220 of 1000\n",
      "building tree 221 of 1000\n",
      "building tree 222 of 1000\n",
      "building tree 223 of 1000\n",
      "building tree 224 of 1000\n",
      "building tree 225 of 1000\n",
      "building tree 226 of 1000\n",
      "building tree 227 of 1000\n",
      "building tree 228 of 1000\n",
      "building tree 229 of 1000\n",
      "building tree 230 of 1000\n",
      "building tree 231 of 1000\n",
      "building tree 232 of 1000\n",
      "building tree 233 of 1000\n",
      "building tree 234 of 1000\n",
      "building tree 235 of 1000\n",
      "building tree 236 of 1000\n",
      "building tree 237 of 1000\n",
      "building tree 238 of 1000\n",
      "building tree 239 of 1000\n",
      "building tree 240 of 1000\n",
      "building tree 241 of 1000\n",
      "building tree 242 of 1000building tree 243 of 1000\n",
      "\n",
      "building tree 244 of 1000\n",
      "building tree 245 of 1000\n",
      "building tree 246 of 1000\n",
      "building tree 247 of 1000\n",
      "building tree 248 of 1000\n",
      "building tree 249 of 1000\n",
      "building tree 250 of 1000\n",
      "building tree 251 of 1000\n",
      "building tree 252 of 1000\n",
      "building tree 253 of 1000\n",
      "building tree 254 of 1000\n",
      "building tree 255 of 1000\n",
      "building tree 256 of 1000\n",
      "building tree 257 of 1000\n",
      "building tree 258 of 1000\n",
      "building tree 259 of 1000\n",
      "building tree 260 of 1000\n",
      "building tree 261 of 1000\n",
      "building tree 262 of 1000\n",
      "building tree 263 of 1000building tree 264 of 1000\n",
      "\n",
      "building tree 265 of 1000\n",
      "building tree 266 of 1000\n",
      "building tree 267 of 1000\n",
      "building tree 268 of 1000\n",
      "building tree 269 of 1000\n",
      "building tree 270 of 1000\n",
      "building tree 271 of 1000\n",
      "building tree 272 of 1000\n",
      "building tree 273 of 1000\n",
      "building tree 274 of 1000\n",
      "building tree 275 of 1000\n",
      "building tree 276 of 1000\n",
      "building tree 277 of 1000\n",
      "building tree 278 of 1000\n",
      "building tree 279 of 1000\n",
      "building tree 280 of 1000\n",
      "building tree 281 of 1000\n",
      "building tree 282 of 1000\n",
      "building tree 283 of 1000\n",
      "building tree 284 of 1000\n",
      "building tree 285 of 1000\n",
      "building tree 286 of 1000\n",
      "building tree 287 of 1000\n",
      "building tree 288 of 1000\n",
      "building tree 289 of 1000\n",
      "building tree 290 of 1000\n",
      "building tree 291 of 1000\n",
      "building tree 292 of 1000\n",
      "building tree 293 of 1000\n",
      "building tree 294 of 1000\n",
      "building tree 295 of 1000\n",
      "building tree 296 of 1000\n",
      "building tree 297 of 1000\n",
      "building tree 298 of 1000\n",
      "building tree 299 of 1000\n",
      "building tree 300 of 1000\n",
      "building tree 301 of 1000\n",
      "building tree 302 of 1000\n",
      "building tree 303 of 1000\n",
      "building tree 304 of 1000\n",
      "building tree 305 of 1000building tree 306 of 1000\n",
      "\n",
      "building tree 307 of 1000\n",
      "building tree 308 of 1000\n",
      "building tree 309 of 1000\n",
      "building tree 310 of 1000\n",
      "building tree 311 of 1000\n",
      "building tree 312 of 1000\n",
      "building tree 313 of 1000\n",
      "building tree 314 of 1000\n",
      "building tree 315 of 1000\n",
      "building tree 316 of 1000\n",
      "building tree 317 of 1000\n",
      "building tree 318 of 1000\n",
      "building tree 319 of 1000\n",
      "building tree 320 of 1000\n",
      "building tree 321 of 1000\n",
      "building tree 322 of 1000\n",
      "building tree 323 of 1000\n",
      "building tree 324 of 1000\n",
      "building tree 325 of 1000\n",
      "building tree 326 of 1000\n",
      "building tree 327 of 1000\n",
      "building tree 328 of 1000\n",
      "building tree 329 of 1000\n",
      "building tree 330 of 1000\n",
      "building tree 331 of 1000\n",
      "building tree 332 of 1000\n",
      "building tree 333 of 1000\n",
      "building tree 334 of 1000\n",
      "building tree 335 of 1000building tree 336 of 1000\n",
      "\n",
      "building tree 337 of 1000\n",
      "building tree 338 of 1000\n",
      "building tree 339 of 1000\n",
      "building tree 340 of 1000\n",
      "building tree 341 of 1000\n",
      "building tree 342 of 1000\n",
      "building tree 343 of 1000\n",
      "building tree 344 of 1000\n",
      "building tree 345 of 1000\n",
      "building tree 346 of 1000\n",
      "building tree 347 of 1000\n",
      "building tree 348 of 1000\n",
      "building tree 349 of 1000\n",
      "building tree 350 of 1000\n",
      "building tree 351 of 1000\n",
      "building tree 352 of 1000\n",
      "building tree 353 of 1000\n",
      "building tree 354 of 1000\n",
      "building tree 355 of 1000\n",
      "building tree 356 of 1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=10)]: Done 345 tasks      | elapsed:   42.2s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "building tree 357 of 1000\n",
      "building tree 358 of 1000\n",
      "building tree 359 of 1000\n",
      "building tree 360 of 1000\n",
      "building tree 361 of 1000\n",
      "building tree 362 of 1000\n",
      "building tree 363 of 1000\n",
      "building tree 364 of 1000\n",
      "building tree 365 of 1000\n",
      "building tree 366 of 1000\n",
      "building tree 367 of 1000\n",
      "building tree 368 of 1000\n",
      "building tree 369 of 1000\n",
      "building tree 370 of 1000\n",
      "building tree 371 of 1000\n",
      "building tree 372 of 1000\n",
      "building tree 373 of 1000\n",
      "building tree 374 of 1000\n",
      "building tree 375 of 1000\n",
      "building tree 376 of 1000\n",
      "building tree 377 of 1000\n",
      "building tree 378 of 1000\n",
      "building tree 379 of 1000\n",
      "building tree 380 of 1000building tree 381 of 1000\n",
      "\n",
      "building tree 382 of 1000\n",
      "building tree 383 of 1000\n",
      "building tree 384 of 1000\n",
      "building tree 385 of 1000\n",
      "building tree 386 of 1000\n",
      "building tree 387 of 1000\n",
      "building tree 388 of 1000\n",
      "building tree 389 of 1000\n",
      "building tree 390 of 1000\n",
      "building tree 391 of 1000\n",
      "building tree 392 of 1000\n",
      "building tree 393 of 1000\n",
      "building tree 394 of 1000building tree 395 of 1000\n",
      "\n",
      "building tree 396 of 1000\n",
      "building tree 397 of 1000\n",
      "building tree 398 of 1000\n",
      "building tree 399 of 1000\n",
      "building tree 400 of 1000\n",
      "building tree 401 of 1000\n",
      "building tree 402 of 1000\n",
      "building tree 403 of 1000\n",
      "building tree 404 of 1000\n",
      "building tree 405 of 1000\n",
      "building tree 406 of 1000\n",
      "building tree 407 of 1000\n",
      "building tree 408 of 1000\n",
      "building tree 409 of 1000\n",
      "building tree 410 of 1000\n",
      "building tree 411 of 1000\n",
      "building tree 412 of 1000\n",
      "building tree 413 of 1000\n",
      "building tree 414 of 1000building tree 415 of 1000\n",
      "\n",
      "building tree 416 of 1000\n",
      "building tree 417 of 1000\n",
      "building tree 418 of 1000\n",
      "building tree 419 of 1000\n",
      "building tree 420 of 1000\n",
      "building tree 421 of 1000\n",
      "building tree 422 of 1000\n",
      "building tree 423 of 1000\n",
      "building tree 424 of 1000\n",
      "building tree 425 of 1000\n",
      "building tree 426 of 1000\n",
      "building tree 427 of 1000\n",
      "building tree 428 of 1000\n",
      "building tree 429 of 1000\n",
      "building tree 430 of 1000\n",
      "building tree 431 of 1000\n",
      "building tree 432 of 1000\n",
      "building tree 433 of 1000\n",
      "building tree 434 of 1000\n",
      "building tree 435 of 1000\n",
      "building tree 436 of 1000\n",
      "building tree 437 of 1000\n",
      "building tree 438 of 1000\n",
      "building tree 439 of 1000\n",
      "building tree 440 of 1000\n",
      "building tree 441 of 1000\n",
      "building tree 442 of 1000\n",
      "building tree 443 of 1000\n",
      "building tree 444 of 1000\n",
      "building tree 445 of 1000\n",
      "building tree 446 of 1000\n",
      "building tree 447 of 1000\n",
      "building tree 448 of 1000\n",
      "building tree 449 of 1000\n",
      "building tree 450 of 1000\n",
      "building tree 451 of 1000\n",
      "building tree 452 of 1000\n",
      "building tree 453 of 1000\n",
      "building tree 454 of 1000\n",
      "building tree 455 of 1000\n",
      "building tree 456 of 1000\n",
      "building tree 457 of 1000\n",
      "building tree 458 of 1000\n",
      "building tree 459 of 1000\n",
      "building tree 460 of 1000\n",
      "building tree 461 of 1000\n",
      "building tree 462 of 1000\n",
      "building tree 463 of 1000\n",
      "building tree 464 of 1000\n",
      "building tree 465 of 1000\n",
      "building tree 466 of 1000\n",
      "building tree 467 of 1000\n",
      "building tree 468 of 1000\n",
      "building tree 469 of 1000\n",
      "building tree 470 of 1000\n",
      "building tree 471 of 1000\n",
      "building tree 472 of 1000\n",
      "building tree 473 of 1000\n",
      "building tree 474 of 1000\n",
      "building tree 475 of 1000\n",
      "building tree 476 of 1000\n",
      "building tree 477 of 1000\n",
      "building tree 478 of 1000\n",
      "building tree 479 of 1000\n",
      "building tree 480 of 1000\n",
      "building tree 481 of 1000\n",
      "building tree 482 of 1000\n",
      "building tree 483 of 1000\n",
      "building tree 484 of 1000\n",
      "building tree 485 of 1000\n",
      "building tree 486 of 1000building tree 487 of 1000\n",
      "\n",
      "building tree 488 of 1000\n",
      "building tree 489 of 1000\n",
      "building tree 490 of 1000\n",
      "building tree 491 of 1000\n",
      "building tree 492 of 1000\n",
      "building tree 493 of 1000\n",
      "building tree 494 of 1000\n",
      "building tree 495 of 1000\n",
      "building tree 496 of 1000\n",
      "building tree 497 of 1000\n",
      "building tree 498 of 1000\n",
      "building tree 499 of 1000\n",
      "building tree 500 of 1000building tree 501 of 1000\n",
      "\n",
      "building tree 502 of 1000\n",
      "building tree 503 of 1000\n",
      "building tree 504 of 1000\n",
      "building tree 505 of 1000\n",
      "building tree 506 of 1000\n",
      "building tree 507 of 1000\n",
      "building tree 508 of 1000\n",
      "building tree 509 of 1000\n",
      "building tree 510 of 1000\n",
      "building tree 511 of 1000\n",
      "building tree 512 of 1000\n",
      "building tree 513 of 1000\n",
      "building tree 514 of 1000\n",
      "building tree 515 of 1000\n",
      "building tree 516 of 1000\n",
      "building tree 517 of 1000\n",
      "building tree 518 of 1000\n",
      "building tree 519 of 1000\n",
      "building tree 520 of 1000\n",
      "building tree 521 of 1000\n",
      "building tree 522 of 1000\n",
      "building tree 523 of 1000\n",
      "building tree 524 of 1000\n",
      "building tree 525 of 1000\n",
      "building tree 526 of 1000\n",
      "building tree 527 of 1000\n",
      "building tree 528 of 1000\n",
      "building tree 529 of 1000\n",
      "building tree 530 of 1000\n",
      "building tree 531 of 1000\n",
      "building tree 532 of 1000\n",
      "building tree 533 of 1000\n",
      "building tree 534 of 1000\n",
      "building tree 535 of 1000\n",
      "building tree 536 of 1000\n",
      "building tree 537 of 1000\n",
      "building tree 538 of 1000\n",
      "building tree 539 of 1000\n",
      "building tree 540 of 1000\n",
      "building tree 541 of 1000\n",
      "building tree 542 of 1000\n",
      "building tree 543 of 1000\n",
      "building tree 544 of 1000\n",
      "building tree 545 of 1000\n",
      "building tree 546 of 1000\n",
      "building tree 547 of 1000\n",
      "building tree 548 of 1000building tree 549 of 1000\n",
      "\n",
      "building tree 550 of 1000\n",
      "building tree 551 of 1000\n",
      "building tree 552 of 1000\n",
      "building tree 553 of 1000\n",
      "building tree 554 of 1000\n",
      "building tree 555 of 1000\n",
      "building tree 556 of 1000\n",
      "building tree 557 of 1000\n",
      "building tree 558 of 1000\n",
      "building tree 559 of 1000\n",
      "building tree 560 of 1000\n",
      "building tree 561 of 1000\n",
      "building tree 562 of 1000\n",
      "building tree 563 of 1000\n",
      "building tree 564 of 1000\n",
      "building tree 565 of 1000\n",
      "building tree 566 of 1000\n",
      "building tree 567 of 1000\n",
      "building tree 568 of 1000\n",
      "building tree 569 of 1000\n",
      "building tree 570 of 1000\n",
      "building tree 571 of 1000\n",
      "building tree 572 of 1000\n",
      "building tree 573 of 1000\n",
      "building tree 574 of 1000\n",
      "building tree 575 of 1000\n",
      "building tree 576 of 1000\n",
      "building tree 577 of 1000\n",
      "building tree 578 of 1000\n",
      "building tree 579 of 1000\n",
      "building tree 580 of 1000\n",
      "building tree 581 of 1000\n",
      "building tree 582 of 1000\n",
      "building tree 583 of 1000\n",
      "building tree 584 of 1000\n",
      "building tree 585 of 1000\n",
      "building tree 586 of 1000\n",
      "building tree 587 of 1000\n",
      "building tree 588 of 1000\n",
      "building tree 589 of 1000\n",
      "building tree 590 of 1000\n",
      "building tree 591 of 1000\n",
      "building tree 592 of 1000\n",
      "building tree 593 of 1000\n",
      "building tree 594 of 1000\n",
      "building tree 595 of 1000\n",
      "building tree 596 of 1000\n",
      "building tree 597 of 1000\n",
      "building tree 598 of 1000\n",
      "building tree 599 of 1000\n",
      "building tree 600 of 1000\n",
      "building tree 601 of 1000\n",
      "building tree 602 of 1000\n",
      "building tree 603 of 1000\n",
      "building tree 604 of 1000\n",
      "building tree 605 of 1000\n",
      "building tree 606 of 1000\n",
      "building tree 607 of 1000\n",
      "building tree 608 of 1000\n",
      "building tree 609 of 1000\n",
      "building tree 610 of 1000\n",
      "building tree 611 of 1000\n",
      "building tree 612 of 1000\n",
      "building tree 613 of 1000\n",
      "building tree 614 of 1000\n",
      "building tree 615 of 1000\n",
      "building tree 616 of 1000\n",
      "building tree 617 of 1000\n",
      "building tree 618 of 1000\n",
      "building tree 619 of 1000\n",
      "building tree 620 of 1000\n",
      "building tree 621 of 1000\n",
      "building tree 622 of 1000building tree 623 of 1000\n",
      "\n",
      "building tree 624 of 1000\n",
      "building tree 625 of 1000\n",
      "building tree 626 of 1000\n",
      "building tree 627 of 1000\n",
      "building tree 628 of 1000\n",
      "building tree 629 of 1000\n",
      "building tree 630 of 1000\n",
      "building tree 631 of 1000\n",
      "building tree 632 of 1000\n",
      "building tree 633 of 1000\n",
      "building tree 634 of 1000\n",
      "building tree 635 of 1000\n",
      "building tree 636 of 1000\n",
      "building tree 637 of 1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=10)]: Done 628 tasks      | elapsed:  1.3min\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "building tree 638 of 1000\n",
      "building tree 639 of 1000\n",
      "building tree 640 of 1000\n",
      "building tree 641 of 1000\n",
      "building tree 642 of 1000\n",
      "building tree 643 of 1000\n",
      "building tree 644 of 1000\n",
      "building tree 645 of 1000\n",
      "building tree 646 of 1000\n",
      "building tree 647 of 1000\n",
      "building tree 648 of 1000\n",
      "building tree 649 of 1000\n",
      "building tree 650 of 1000\n",
      "building tree 651 of 1000\n",
      "building tree 652 of 1000\n",
      "building tree 653 of 1000\n",
      "building tree 654 of 1000\n",
      "building tree 655 of 1000\n",
      "building tree 656 of 1000\n",
      "building tree 657 of 1000\n",
      "building tree 658 of 1000\n",
      "building tree 659 of 1000\n",
      "building tree 660 of 1000\n",
      "building tree 661 of 1000\n",
      "building tree 662 of 1000\n",
      "building tree 663 of 1000\n",
      "building tree 664 of 1000\n",
      "building tree 665 of 1000\n",
      "building tree 666 of 1000\n",
      "building tree 667 of 1000\n",
      "building tree 668 of 1000\n",
      "building tree 669 of 1000\n",
      "building tree 670 of 1000\n",
      "building tree 671 of 1000\n",
      "building tree 672 of 1000\n",
      "building tree 673 of 1000\n",
      "building tree 674 of 1000\n",
      "building tree 675 of 1000\n",
      "building tree 676 of 1000\n",
      "building tree 677 of 1000\n",
      "building tree 678 of 1000\n",
      "building tree 679 of 1000\n",
      "building tree 680 of 1000\n",
      "building tree 681 of 1000\n",
      "building tree 682 of 1000\n",
      "building tree 683 of 1000building tree 684 of 1000\n",
      "\n",
      "building tree 685 of 1000\n",
      "building tree 686 of 1000\n",
      "building tree 687 of 1000\n",
      "building tree 688 of 1000\n",
      "building tree 689 of 1000\n",
      "building tree 690 of 1000\n",
      "building tree 691 of 1000building tree 692 of 1000\n",
      "building tree 693 of 1000\n",
      "building tree 694 of 1000\n",
      "\n",
      "building tree 695 of 1000\n",
      "building tree 696 of 1000\n",
      "building tree 697 of 1000\n",
      "building tree 698 of 1000\n",
      "building tree 699 of 1000\n",
      "building tree 700 of 1000building tree 701 of 1000\n",
      "\n",
      "building tree 702 of 1000\n",
      "building tree 703 of 1000\n",
      "building tree 704 of 1000\n",
      "building tree 705 of 1000\n",
      "building tree 706 of 1000\n",
      "building tree 707 of 1000\n",
      "building tree 708 of 1000\n",
      "building tree 709 of 1000\n",
      "building tree 710 of 1000\n",
      "building tree 711 of 1000\n",
      "building tree 712 of 1000\n",
      "building tree 713 of 1000\n",
      "building tree 714 of 1000\n",
      "building tree 715 of 1000\n",
      "building tree 716 of 1000\n",
      "building tree 717 of 1000\n",
      "building tree 718 of 1000\n",
      "building tree 719 of 1000\n",
      "building tree 720 of 1000\n",
      "building tree 721 of 1000\n",
      "building tree 722 of 1000building tree 723 of 1000\n",
      "\n",
      "building tree 724 of 1000\n",
      "building tree 725 of 1000\n",
      "building tree 726 of 1000\n",
      "building tree 727 of 1000\n",
      "building tree 728 of 1000\n",
      "building tree 729 of 1000\n",
      "building tree 730 of 1000\n",
      "building tree 731 of 1000\n",
      "building tree 732 of 1000\n",
      "building tree 733 of 1000\n",
      "building tree 734 of 1000\n",
      "building tree 735 of 1000\n",
      "building tree 736 of 1000\n",
      "building tree 737 of 1000\n",
      "building tree 738 of 1000\n",
      "building tree 739 of 1000\n",
      "building tree 740 of 1000\n",
      "building tree 741 of 1000\n",
      "building tree 742 of 1000\n",
      "building tree 743 of 1000\n",
      "building tree 744 of 1000\n",
      "building tree 745 of 1000\n",
      "building tree 746 of 1000\n",
      "building tree 747 of 1000\n",
      "building tree 748 of 1000\n",
      "building tree 749 of 1000\n",
      "building tree 750 of 1000\n",
      "building tree 751 of 1000\n",
      "building tree 752 of 1000\n",
      "building tree 753 of 1000\n",
      "building tree 754 of 1000\n",
      "building tree 755 of 1000\n",
      "building tree 756 of 1000\n",
      "building tree 757 of 1000\n",
      "building tree 758 of 1000\n",
      "building tree 759 of 1000\n",
      "building tree 760 of 1000\n",
      "building tree 761 of 1000\n",
      "building tree 762 of 1000\n",
      "building tree 763 of 1000\n",
      "building tree 764 of 1000\n",
      "building tree 765 of 1000\n",
      "building tree 766 of 1000\n",
      "building tree 767 of 1000\n",
      "building tree 768 of 1000\n",
      "building tree 769 of 1000\n",
      "building tree 770 of 1000\n",
      "building tree 771 of 1000\n",
      "building tree 772 of 1000\n",
      "building tree 773 of 1000\n",
      "building tree 774 of 1000\n",
      "building tree 775 of 1000\n",
      "building tree 776 of 1000\n",
      "building tree 777 of 1000\n",
      "building tree 778 of 1000\n",
      "building tree 779 of 1000\n",
      "building tree 780 of 1000\n",
      "building tree 781 of 1000\n",
      "building tree 782 of 1000\n",
      "building tree 783 of 1000\n",
      "building tree 784 of 1000\n",
      "building tree 785 of 1000\n",
      "building tree 786 of 1000\n",
      "building tree 787 of 1000\n",
      "building tree 788 of 1000\n",
      "building tree 789 of 1000\n",
      "building tree 790 of 1000\n",
      "building tree 791 of 1000\n",
      "building tree 792 of 1000\n",
      "building tree 793 of 1000\n",
      "building tree 794 of 1000\n",
      "building tree 795 of 1000\n",
      "building tree 796 of 1000\n",
      "building tree 797 of 1000\n",
      "building tree 798 of 1000\n",
      "building tree 799 of 1000\n",
      "building tree 800 of 1000\n",
      "building tree 801 of 1000\n",
      "building tree 802 of 1000\n",
      "building tree 803 of 1000\n",
      "building tree 804 of 1000\n",
      "building tree 805 of 1000\n",
      "building tree 806 of 1000\n",
      "building tree 807 of 1000\n",
      "building tree 808 of 1000\n",
      "building tree 809 of 1000\n",
      "building tree 810 of 1000\n",
      "building tree 811 of 1000\n",
      "building tree 812 of 1000\n",
      "building tree 813 of 1000\n",
      "building tree 814 of 1000\n",
      "building tree 815 of 1000\n",
      "building tree 816 of 1000\n",
      "building tree 817 of 1000\n",
      "building tree 818 of 1000\n",
      "building tree 819 of 1000\n",
      "building tree 820 of 1000\n",
      "building tree 821 of 1000\n",
      "building tree 822 of 1000\n",
      "building tree 823 of 1000\n",
      "building tree 824 of 1000\n",
      "building tree 825 of 1000\n",
      "building tree 826 of 1000\n",
      "building tree 827 of 1000\n",
      "building tree 828 of 1000\n",
      "building tree 829 of 1000\n",
      "building tree 830 of 1000\n",
      "building tree 831 of 1000\n",
      "building tree 832 of 1000\n",
      "building tree 833 of 1000\n",
      "building tree 834 of 1000\n",
      "building tree 835 of 1000\n",
      "building tree 836 of 1000\n",
      "building tree 837 of 1000building tree 838 of 1000\n",
      "\n",
      "building tree 839 of 1000\n",
      "building tree 840 of 1000\n",
      "building tree 841 of 1000\n",
      "building tree 842 of 1000\n",
      "building tree 843 of 1000\n",
      "building tree 844 of 1000\n",
      "building tree 845 of 1000\n",
      "building tree 846 of 1000\n",
      "building tree 847 of 1000\n",
      "building tree 848 of 1000\n",
      "building tree 849 of 1000\n",
      "building tree 850 of 1000\n",
      "building tree 851 of 1000\n",
      "building tree 852 of 1000\n",
      "building tree 853 of 1000\n",
      "building tree 854 of 1000\n",
      "building tree 855 of 1000\n",
      "building tree 856 of 1000\n",
      "building tree 857 of 1000building tree 858 of 1000\n",
      "\n",
      "building tree 859 of 1000\n",
      "building tree 860 of 1000\n",
      "building tree 861 of 1000\n",
      "building tree 862 of 1000\n",
      "building tree 863 of 1000\n",
      "building tree 864 of 1000\n",
      "building tree 865 of 1000\n",
      "building tree 866 of 1000\n",
      "building tree 867 of 1000\n",
      "building tree 868 of 1000\n",
      "building tree 869 of 1000\n",
      "building tree 870 of 1000\n",
      "building tree 871 of 1000\n",
      "building tree 872 of 1000\n",
      "building tree 873 of 1000\n",
      "building tree 874 of 1000\n",
      "building tree 875 of 1000\n",
      "building tree 876 of 1000\n",
      "building tree 877 of 1000\n",
      "building tree 878 of 1000\n",
      "building tree 879 of 1000building tree 880 of 1000\n",
      "\n",
      "building tree 881 of 1000\n",
      "building tree 882 of 1000\n",
      "building tree 883 of 1000\n",
      "building tree 884 of 1000\n",
      "building tree 885 of 1000\n",
      "building tree 886 of 1000\n",
      "building tree 887 of 1000building tree 888 of 1000\n",
      "\n",
      "building tree 889 of 1000\n",
      "building tree 890 of 1000\n",
      "building tree 891 of 1000\n",
      "building tree 892 of 1000\n",
      "building tree 893 of 1000\n",
      "building tree 894 of 1000\n",
      "building tree 895 of 1000\n",
      "building tree 896 of 1000\n",
      "building tree 897 of 1000\n",
      "building tree 898 of 1000\n",
      "building tree 899 of 1000\n",
      "building tree 900 of 1000\n",
      "building tree 901 of 1000\n",
      "building tree 902 of 1000\n",
      "building tree 903 of 1000\n",
      "building tree 904 of 1000\n",
      "building tree 905 of 1000\n",
      "building tree 906 of 1000building tree 907 of 1000\n",
      "\n",
      "building tree 908 of 1000\n",
      "building tree 909 of 1000\n",
      "building tree 910 of 1000\n",
      "building tree 911 of 1000\n",
      "building tree 912 of 1000\n",
      "building tree 913 of 1000\n",
      "building tree 914 of 1000\n",
      "building tree 915 of 1000\n",
      "building tree 916 of 1000\n",
      "building tree 917 of 1000\n",
      "building tree 918 of 1000\n",
      "building tree 919 of 1000\n",
      "building tree 920 of 1000\n",
      "building tree 921 of 1000\n",
      "building tree 922 of 1000\n",
      "building tree 923 of 1000\n",
      "building tree 924 of 1000\n",
      "building tree 925 of 1000\n",
      "building tree 926 of 1000\n",
      "building tree 927 of 1000\n",
      "building tree 928 of 1000\n",
      "building tree 929 of 1000\n",
      "building tree 930 of 1000\n",
      "building tree 931 of 1000\n",
      "building tree 932 of 1000\n",
      "building tree 933 of 1000\n",
      "building tree 934 of 1000\n",
      "building tree 935 of 1000\n",
      "building tree 936 of 1000\n",
      "building tree 937 of 1000\n",
      "building tree 938 of 1000\n",
      "building tree 939 of 1000\n",
      "building tree 940 of 1000\n",
      "building tree 941 of 1000\n",
      "building tree 942 of 1000\n",
      "building tree 943 of 1000\n",
      "building tree 944 of 1000\n",
      "building tree 945 of 1000\n",
      "building tree 946 of 1000\n",
      "building tree 947 of 1000\n",
      "building tree 948 of 1000\n",
      "building tree 949 of 1000\n",
      "building tree 950 of 1000\n",
      "building tree 951 of 1000\n",
      "building tree 952 of 1000\n",
      "building tree 953 of 1000\n",
      "building tree 954 of 1000\n",
      "building tree 955 of 1000\n",
      "building tree 956 of 1000\n",
      "building tree 957 of 1000\n",
      "building tree 958 of 1000\n",
      "building tree 959 of 1000\n",
      "building tree 960 of 1000\n",
      "building tree 961 of 1000\n",
      "building tree 962 of 1000\n",
      "building tree 963 of 1000\n",
      "building tree 964 of 1000\n",
      "building tree 965 of 1000\n",
      "building tree 966 of 1000\n",
      "building tree 967 of 1000\n",
      "building tree 968 of 1000\n",
      "building tree 969 of 1000\n",
      "building tree 970 of 1000\n",
      "building tree 971 of 1000\n",
      "building tree 972 of 1000\n",
      "building tree 973 of 1000\n",
      "building tree 974 of 1000building tree 975 of 1000\n",
      "\n",
      "building tree 976 of 1000\n",
      "building tree 977 of 1000\n",
      "building tree 978 of 1000\n",
      "building tree 979 of 1000\n",
      "building tree 980 of 1000\n",
      "building tree 981 of 1000\n",
      "building tree 982 of 1000\n",
      "building tree 983 of 1000\n",
      "building tree 984 of 1000\n",
      "building tree 985 of 1000\n",
      "building tree 986 of 1000\n",
      "building tree 987 of 1000\n",
      "building tree 988 of 1000\n",
      "building tree 989 of 1000\n",
      "building tree 990 of 1000\n",
      "building tree 991 of 1000\n",
      "building tree 992 of 1000\n",
      "building tree 993 of 1000\n",
      "building tree 994 of 1000\n",
      "building tree 995 of 1000\n",
      "building tree 996 of 1000\n",
      "building tree 997 of 1000\n",
      "building tree 998 of 1000\n",
      "building tree 999 of 1000\n",
      "building tree 1000 of 1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=10)]: Done 1000 out of 1000 | elapsed:  2.0min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "                       max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
       "                       min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "                       min_samples_leaf=1, min_samples_split=2,\n",
       "                       min_weight_fraction_leaf=0.0, n_estimators=1000,\n",
       "                       n_jobs=10, oob_score=False, random_state=None, verbose=2,\n",
       "                       warm_start=False)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = RandomForestClassifier(n_estimators=1000,verbose = 2,n_jobs=10)\n",
    "model.fit(x_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=10)]: Using backend ThreadingBackend with 10 concurrent workers.\n",
      "[Parallel(n_jobs=10)]: Done  21 tasks      | elapsed:    0.0s\n",
      "[Parallel(n_jobs=10)]: Done 142 tasks      | elapsed:    0.1s\n",
      "[Parallel(n_jobs=10)]: Done 345 tasks      | elapsed:    0.3s\n",
      "[Parallel(n_jobs=10)]: Done 628 tasks      | elapsed:    0.6s\n",
      "[Parallel(n_jobs=10)]: Done 1000 out of 1000 | elapsed:    0.9s finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.9216417910447762"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.score(x_test,y_test) #prev 0.91544"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "#features_importance = list(model.feature_importances_)\n",
    "#print(np.max(features_importance))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'features_importance' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-26-c006d75a3d10>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfeatures_importance\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'features_importance' is not defined"
     ]
    }
   ],
   "source": [
    "print(np.max(features_importance))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "_kg_hide-output": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=10)]: Using backend ThreadingBackend with 10 concurrent workers.\n",
      "[Parallel(n_jobs=10)]: Done  21 tasks      | elapsed:    0.0s\n",
      "[Parallel(n_jobs=10)]: Done 142 tasks      | elapsed:    0.1s\n",
      "[Parallel(n_jobs=10)]: Done 345 tasks      | elapsed:    0.3s\n",
      "[Parallel(n_jobs=10)]: Done 628 tasks      | elapsed:    0.6s\n",
      "[Parallel(n_jobs=10)]: Done 1000 out of 1000 | elapsed:    0.9s finished\n"
     ]
    }
   ],
   "source": [
    "rm_df = thershold_matrix(model,x_test,y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Threshold</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Recall</th>\n",
       "      <th>F1</th>\n",
       "      <th>TN</th>\n",
       "      <th>FP</th>\n",
       "      <th>FN</th>\n",
       "      <th>TP</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>0.45</td>\n",
       "      <td>0.920983</td>\n",
       "      <td>0.932203</td>\n",
       "      <td>0.447154</td>\n",
       "      <td>0.908632</td>\n",
       "      <td>3921.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>340.0</td>\n",
       "      <td>275.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Threshold  Accuracy  Precision    Recall        F1      TN    FP     FN  \\\n",
       "8       0.45  0.920983   0.932203  0.447154  0.908632  3921.0  20.0  340.0   \n",
       "\n",
       "      TP  \n",
       "8  275.0  "
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rm_df[rm_df['F1']==rm_df.F1.max()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=10)]: Using backend ThreadingBackend with 10 concurrent workers.\n",
      "[Parallel(n_jobs=10)]: Done  21 tasks      | elapsed:    0.1s\n",
      "[Parallel(n_jobs=10)]: Done 142 tasks      | elapsed:    0.5s\n",
      "[Parallel(n_jobs=10)]: Done 345 tasks      | elapsed:    1.1s\n",
      "[Parallel(n_jobs=10)]: Done 628 tasks      | elapsed:    2.0s\n",
      "[Parallel(n_jobs=10)]: Done 1000 out of 1000 | elapsed:    3.1s finished\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<a href='RandomForest_90_.csv' target='_blank'>RandomForest_90_.csv</a><br>"
      ],
      "text/plain": [
       "/kaggle/working/RandomForest_90_.csv"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_proba_df = pd.DataFrame(model.predict_proba(test_null_removed))\n",
    "Y_test_pred = pred_proba_df.applymap(lambda x: 1 if x>0.45 else 0)\n",
    "y_pred = Y_test_pred.iloc[:,1]\n",
    "X = test.Col1\n",
    "df = pd.DataFrame({'Col1': X,'Col2': y_pred})\n",
    "df.to_csv('RandomForest_90_.csv',index= False)\n",
    "from IPython.display import FileLink\n",
    "FileLink(r'RandomForest_90_.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'RandomForestRegressor' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-30-4e304ece943f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      4\u001b[0m }\n\u001b[1;32m      5\u001b[0m \u001b[0;31m# Create a based model\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0mrf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mRandomForestRegressor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m \u001b[0;31m# Instantiate the grid search model\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m grid_search = GridSearchCV(estimator = rf, param_grid = param_grid, \n",
      "\u001b[0;31mNameError\u001b[0m: name 'RandomForestRegressor' is not defined"
     ]
    }
   ],
   "source": [
    "param_grid = {\n",
    "    'bootstrap': [True],\n",
    "    'n_estimators': [100, 200, 300, 1000]\n",
    "}\n",
    "# Create a based model\n",
    "rf = RandomForestRegressor()\n",
    "# Instantiate the grid search model\n",
    "grid_search = GridSearchCV(estimator = rf, param_grid = param_grid, \n",
    "                          cv = 3, n_jobs = -1, verbose = 2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Confusion matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy Score : 0.8612818261633012\n",
      "Precision Score : 0.34545454545454546\n",
      "Recall Score : 0.030894308943089432\n",
      "F1 Score : 0.056716417910447764\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score,recall_score,precision_score,f1_score\n",
    "print('Accuracy Score : ' + str(accuracy_score(y_test,lg_fs.predict(x_test))))\n",
    "print('Precision Score : ' + str(precision_score(y_test,lg_fs.predict(x_test))))\n",
    "print('Recall Score : ' + str(recall_score(y_test,lg_fs.predict(x_test))))\n",
    "print('F1 Score : ' + str(f1_score(y_test,lg_fs.predict(x_test))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion Matrix : \n",
      "[[3905   36]\n",
      " [ 596   19]]\n"
     ]
    }
   ],
   "source": [
    "\n",
    "print('Confusion Matrix : \\n' + str(confusion_matrix(y_test,lg_fs.predict(x_test))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.decomposition import PCA\n",
    "pca = PCA(n_components=3)\n",
    "x = pca.fit_transform(train_null_removed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.96093411, 0.02969953, 0.00500398])"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "explained_variance=pca.explained_variance_ratio_\n",
    "explained_variance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.995637615623037"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "explained_variance.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train,x_test,y_train,y_test = train_test_split(x,target,test_size = .30, random_state = 24,stratify=target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "lg_PCA = LogisticRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "                   intercept_scaling=1, l1_ratio=None, max_iter=100,\n",
       "                   multi_class='warn', n_jobs=None, penalty='l2',\n",
       "                   random_state=None, solver='warn', tol=0.0001, verbose=0,\n",
       "                   warm_start=False)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lg_PCA.fit(x_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.4020486555697823"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lg_PCA.score(x_test,y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "logistic_PCA_score_df = thershold_matrix(lg_PCA,x_test,y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Threshold</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Recall</th>\n",
       "      <th>F1</th>\n",
       "      <th>TN</th>\n",
       "      <th>FP</th>\n",
       "      <th>FN</th>\n",
       "      <th>TP</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>0.65</td>\n",
       "      <td>0.864094</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.006775</td>\n",
       "      <td>0.803673</td>\n",
       "      <td>4719.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>733.0</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Threshold  Accuracy  Precision    Recall        F1      TN    FP     FN  \\\n",
       "12       0.65  0.864094   0.333333  0.006775  0.803673  4719.0  10.0  733.0   \n",
       "\n",
       "     TP  \n",
       "12  5.0  "
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logistic_PCA_score_df[logistic_PCA_score_df['F1']==logistic_PCA_score_df.F1.max()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Threshold</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Recall</th>\n",
       "      <th>F1</th>\n",
       "      <th>TN</th>\n",
       "      <th>FP</th>\n",
       "      <th>FN</th>\n",
       "      <th>TP</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>0.65</td>\n",
       "      <td>0.864094</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.006775</td>\n",
       "      <td>0.803673</td>\n",
       "      <td>4719.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>733.0</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>17</td>\n",
       "      <td>0.90</td>\n",
       "      <td>0.865008</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.004065</td>\n",
       "      <td>0.803450</td>\n",
       "      <td>4726.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>735.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>0.60</td>\n",
       "      <td>0.863545</td>\n",
       "      <td>0.277778</td>\n",
       "      <td>0.006775</td>\n",
       "      <td>0.803393</td>\n",
       "      <td>4716.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>733.0</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>0.70</td>\n",
       "      <td>0.864094</td>\n",
       "      <td>0.307692</td>\n",
       "      <td>0.005420</td>\n",
       "      <td>0.803331</td>\n",
       "      <td>4720.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>734.0</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>16</td>\n",
       "      <td>0.85</td>\n",
       "      <td>0.864459</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.004065</td>\n",
       "      <td>0.803172</td>\n",
       "      <td>4723.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>735.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>0.55</td>\n",
       "      <td>0.862447</td>\n",
       "      <td>0.230769</td>\n",
       "      <td>0.008130</td>\n",
       "      <td>0.803167</td>\n",
       "      <td>4709.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>732.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>19</td>\n",
       "      <td>0.99</td>\n",
       "      <td>0.865008</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.002710</td>\n",
       "      <td>0.803101</td>\n",
       "      <td>4727.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>736.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>0.75</td>\n",
       "      <td>0.864277</td>\n",
       "      <td>0.300000</td>\n",
       "      <td>0.004065</td>\n",
       "      <td>0.803079</td>\n",
       "      <td>4722.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>735.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>0.80</td>\n",
       "      <td>0.864277</td>\n",
       "      <td>0.300000</td>\n",
       "      <td>0.004065</td>\n",
       "      <td>0.803079</td>\n",
       "      <td>4722.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>735.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>18</td>\n",
       "      <td>0.95</td>\n",
       "      <td>0.864825</td>\n",
       "      <td>0.400000</td>\n",
       "      <td>0.002710</td>\n",
       "      <td>0.803009</td>\n",
       "      <td>4726.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>736.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.402049</td>\n",
       "      <td>0.155083</td>\n",
       "      <td>0.771003</td>\n",
       "      <td>0.466634</td>\n",
       "      <td>1629.0</td>\n",
       "      <td>3100.0</td>\n",
       "      <td>169.0</td>\n",
       "      <td>569.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>0.45</td>\n",
       "      <td>0.159868</td>\n",
       "      <td>0.136937</td>\n",
       "      <td>0.985095</td>\n",
       "      <td>0.084497</td>\n",
       "      <td>147.0</td>\n",
       "      <td>4582.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>727.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0.40</td>\n",
       "      <td>0.145784</td>\n",
       "      <td>0.135926</td>\n",
       "      <td>0.994580</td>\n",
       "      <td>0.055011</td>\n",
       "      <td>63.0</td>\n",
       "      <td>4666.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>734.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>0.35</td>\n",
       "      <td>0.139930</td>\n",
       "      <td>0.135259</td>\n",
       "      <td>0.995935</td>\n",
       "      <td>0.043050</td>\n",
       "      <td>30.0</td>\n",
       "      <td>4699.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>735.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.30</td>\n",
       "      <td>0.139382</td>\n",
       "      <td>0.135453</td>\n",
       "      <td>0.998645</td>\n",
       "      <td>0.041298</td>\n",
       "      <td>25.0</td>\n",
       "      <td>4704.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>737.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.138101</td>\n",
       "      <td>0.135279</td>\n",
       "      <td>0.998645</td>\n",
       "      <td>0.038724</td>\n",
       "      <td>18.0</td>\n",
       "      <td>4711.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>737.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.137370</td>\n",
       "      <td>0.135180</td>\n",
       "      <td>0.998645</td>\n",
       "      <td>0.037250</td>\n",
       "      <td>14.0</td>\n",
       "      <td>4715.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>737.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.15</td>\n",
       "      <td>0.136821</td>\n",
       "      <td>0.135105</td>\n",
       "      <td>0.998645</td>\n",
       "      <td>0.036143</td>\n",
       "      <td>11.0</td>\n",
       "      <td>4718.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>737.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.10</td>\n",
       "      <td>0.136272</td>\n",
       "      <td>0.135031</td>\n",
       "      <td>0.998645</td>\n",
       "      <td>0.035035</td>\n",
       "      <td>8.0</td>\n",
       "      <td>4721.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>737.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.135906</td>\n",
       "      <td>0.135115</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.033964</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4724.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>738.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Threshold  Accuracy  Precision    Recall        F1      TN      FP     FN  \\\n",
       "12       0.65  0.864094   0.333333  0.006775  0.803673  4719.0    10.0  733.0   \n",
       "17       0.90  0.865008   0.500000  0.004065  0.803450  4726.0     3.0  735.0   \n",
       "11       0.60  0.863545   0.277778  0.006775  0.803393  4716.0    13.0  733.0   \n",
       "13       0.70  0.864094   0.307692  0.005420  0.803331  4720.0     9.0  734.0   \n",
       "16       0.85  0.864459   0.333333  0.004065  0.803172  4723.0     6.0  735.0   \n",
       "10       0.55  0.862447   0.230769  0.008130  0.803167  4709.0    20.0  732.0   \n",
       "19       0.99  0.865008   0.500000  0.002710  0.803101  4727.0     2.0  736.0   \n",
       "14       0.75  0.864277   0.300000  0.004065  0.803079  4722.0     7.0  735.0   \n",
       "15       0.80  0.864277   0.300000  0.004065  0.803079  4722.0     7.0  735.0   \n",
       "18       0.95  0.864825   0.400000  0.002710  0.803009  4726.0     3.0  736.0   \n",
       "9        0.50  0.402049   0.155083  0.771003  0.466634  1629.0  3100.0  169.0   \n",
       "8        0.45  0.159868   0.136937  0.985095  0.084497   147.0  4582.0   11.0   \n",
       "7        0.40  0.145784   0.135926  0.994580  0.055011    63.0  4666.0    4.0   \n",
       "6        0.35  0.139930   0.135259  0.995935  0.043050    30.0  4699.0    3.0   \n",
       "5        0.30  0.139382   0.135453  0.998645  0.041298    25.0  4704.0    1.0   \n",
       "4        0.25  0.138101   0.135279  0.998645  0.038724    18.0  4711.0    1.0   \n",
       "3        0.20  0.137370   0.135180  0.998645  0.037250    14.0  4715.0    1.0   \n",
       "2        0.15  0.136821   0.135105  0.998645  0.036143    11.0  4718.0    1.0   \n",
       "1        0.10  0.136272   0.135031  0.998645  0.035035     8.0  4721.0    1.0   \n",
       "0        0.05  0.135906   0.135115  1.000000  0.033964     5.0  4724.0    0.0   \n",
       "\n",
       "       TP  \n",
       "12    5.0  \n",
       "17    3.0  \n",
       "11    5.0  \n",
       "13    4.0  \n",
       "16    3.0  \n",
       "10    6.0  \n",
       "19    2.0  \n",
       "14    3.0  \n",
       "15    3.0  \n",
       "18    2.0  \n",
       "9   569.0  \n",
       "8   727.0  \n",
       "7   734.0  \n",
       "6   735.0  \n",
       "5   737.0  \n",
       "4   737.0  \n",
       "3   737.0  \n",
       "2   737.0  \n",
       "1   737.0  \n",
       "0   738.0  "
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logistic_PCA_score_df.sort_values('F1',ascending =False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion Matrix : \n",
      "[[1629 3100]\n",
      " [ 169  569]]\n"
     ]
    }
   ],
   "source": [
    "print('Confusion Matrix : \\n' + str(confusion_matrix(y_test,lg_PCA.predict(x_test))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_test = pca.fit_transform(test_null_removed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(20442, 3)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = lg_PCA.predict(x_test)\n",
    "X = test.Col1\n",
    "df = pd.DataFrame({'Col1': X,'Col2': y_pred})\n",
    "df.to_csv('Linear_regress_PCA_Submission.csv',index= False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(18221, 9605)\n",
      "(20442, 9605)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "train_objs_num = len(train_null_removed)\n",
    "\n",
    "dataset = pd.concat(objs=[train_null_removed, test_null_removed], axis=0)\n",
    "dataset_preprocessed = pd.get_dummies(dataset,columns = cat_column)\n",
    "train_preprocessed = dataset_preprocessed[:train_objs_num]\n",
    "test_preprocessed = dataset_preprocessed[train_objs_num:]\n",
    "print(train_preprocessed.shape)\n",
    "print(test_preprocessed.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import MinMaxScaler\n",
    "scaler = MinMaxScaler()\n",
    "train_preprocessed[cont_column] = scaler.fit_transform(train_preprocessed[cont_column])\n",
    "test_preprocessed[cont_column] = scaler.fit_transform(test_preprocessed[cont_column])\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train,x_test,y_train,y_test = train_test_split(train_preprocessed,target,test_size = .25, random_state = 24,stratify=target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "                   intercept_scaling=1, l1_ratio=None, max_iter=100,\n",
       "                   multi_class='warn', n_jobs=None, penalty='l2',\n",
       "                   random_state=None, solver='warn', tol=0.0001, verbose=0,\n",
       "                   warm_start=False)"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lg_preprocessed = LogisticRegression()\n",
    "lg_preprocessed.fit(x_train,y_train)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8485513608428445"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lg_preprocessed.score(x_test,y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "lg_preprocessed_df = thershold_matrix(lg_preprocessed,x_test,y_test)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Threshold</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Recall</th>\n",
       "      <th>F1</th>\n",
       "      <th>TN</th>\n",
       "      <th>FP</th>\n",
       "      <th>FN</th>\n",
       "      <th>TP</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.848551</td>\n",
       "      <td>0.424547</td>\n",
       "      <td>0.343089</td>\n",
       "      <td>0.841633</td>\n",
       "      <td>3655.0</td>\n",
       "      <td>286.0</td>\n",
       "      <td>404.0</td>\n",
       "      <td>211.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Threshold  Accuracy  Precision    Recall        F1      TN     FP     FN  \\\n",
       "9        0.5  0.848551   0.424547  0.343089  0.841633  3655.0  286.0  404.0   \n",
       "\n",
       "      TP  \n",
       "9  211.0  "
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lg_preprocessed_df[lg_preprocessed_df['F1']==lg_preprocessed_df.F1.max()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<a href='Linear_regress_HyperParam_Submission_90_.csv' target='_blank'>Linear_regress_HyperParam_Submission_90_.csv</a><br>"
      ],
      "text/plain": [
       "/kaggle/working/Linear_regress_HyperParam_Submission_90_.csv"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#y_pred = lg_preprocessed.predict(test_preprocessed)\n",
    "pred_proba_df = pd.DataFrame(lg_preprocessed.predict_proba(test_preprocessed))\n",
    "Y_test_pred = pred_proba_df.applymap(lambda x: 1 if x>0.5 else 0)\n",
    "y_pred = Y_test_pred.iloc[:,1]\n",
    "X = test.Col1\n",
    "df = pd.DataFrame({'Col1': X,'Col2': y_pred})\n",
    "df.to_csv('Linear_regress_HyperParam_Submission_90_.csv',index= False)\n",
    "from IPython.display import FileLink\n",
    "FileLink(r'Linear_regress_HyperParam_Submission_90_.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion Matrix : \n",
      "[[3655  286]\n",
      " [ 404  211]]\n"
     ]
    }
   ],
   "source": [
    "print('Confusion Matrix : \\n' + str(confusion_matrix(y_test,lg_preprocessed.predict(x_test))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=0.1, class_weight=None, dual=False, fit_intercept=True,\n",
       "                   intercept_scaling=1, l1_ratio=None, max_iter=100,\n",
       "                   multi_class='warn', n_jobs=None, penalty='l2',\n",
       "                   random_state=None, solver='warn', tol=0.0001, verbose=0,\n",
       "                   warm_start=False)"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lgs_clf_hyperparam = LogisticRegression(penalty='l2', C=0.1)\n",
    "lgs_clf_hyperparam.fit(x_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8667690956979807"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lgs_clf_hyperparam.score(x_test,y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "lgs_clf_hyperparam_df = thershold_matrix(lgs_clf_hyperparam,x_test,y_test)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Threshold</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Recall</th>\n",
       "      <th>F1</th>\n",
       "      <th>TN</th>\n",
       "      <th>FP</th>\n",
       "      <th>FN</th>\n",
       "      <th>TP</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.866769</td>\n",
       "      <td>0.513699</td>\n",
       "      <td>0.243902</td>\n",
       "      <td>0.845668</td>\n",
       "      <td>3799.0</td>\n",
       "      <td>142.0</td>\n",
       "      <td>465.0</td>\n",
       "      <td>150.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Threshold  Accuracy  Precision    Recall        F1      TN     FP     FN  \\\n",
       "9        0.5  0.866769   0.513699  0.243902  0.845668  3799.0  142.0  465.0   \n",
       "\n",
       "      TP  \n",
       "9  150.0  "
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "lgs_clf_hyperparam_df[lgs_clf_hyperparam_df['F1']==lgs_clf_hyperparam_df.F1.max()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Threshold</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Recall</th>\n",
       "      <th>F1</th>\n",
       "      <th>TN</th>\n",
       "      <th>FP</th>\n",
       "      <th>FN</th>\n",
       "      <th>TP</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.866769</td>\n",
       "      <td>0.513699</td>\n",
       "      <td>0.243902</td>\n",
       "      <td>0.845668</td>\n",
       "      <td>3799.0</td>\n",
       "      <td>142.0</td>\n",
       "      <td>465.0</td>\n",
       "      <td>150.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Threshold  Accuracy  Precision    Recall        F1      TN     FP     FN  \\\n",
       "9        0.5  0.866769   0.513699  0.243902  0.845668  3799.0  142.0  465.0   \n",
       "\n",
       "      TP  \n",
       "9  150.0  "
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lgs_clf_hyperparam_df[lgs_clf_hyperparam_df['Threshold']==0.50]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<a href='Linear_regress_HyperParam_Submission.csv' target='_blank'>Linear_regress_HyperParam_Submission.csv</a><br>"
      ],
      "text/plain": [
       "/kaggle/working/Linear_regress_HyperParam_Submission.csv"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = lgs_clf_hyperparam.predict(test_preprocessed)\n",
    "X = test.Col1\n",
    "df = pd.DataFrame({'Col1': X,'Col2': y_pred})\n",
    "df.to_csv('Linear_regress_HyperParam_Submission.csv',index= False)\n",
    "FileLink(r'Linear_regress_HyperParam_Submission.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics.f1_score(y_test,y_test,average='weighted')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Learning rate:  0.05\n",
      "Accuracy score (training): 0.865\n",
      "Accuracy score (validation): 0.865\n",
      "Learning rate:  0.075\n",
      "Accuracy score (training): 0.865\n",
      "Accuracy score (validation): 0.865\n",
      "Learning rate:  0.1\n",
      "Accuracy score (training): 0.865\n",
      "Accuracy score (validation): 0.865\n",
      "Learning rate:  0.25\n",
      "Accuracy score (training): 0.865\n",
      "Accuracy score (validation): 0.865\n",
      "Learning rate:  0.5\n",
      "Accuracy score (training): 0.866\n",
      "Accuracy score (validation): 0.865\n",
      "Learning rate:  0.75\n",
      "Accuracy score (training): 0.866\n",
      "Accuracy score (validation): 0.863\n",
      "Learning rate:  1\n",
      "Accuracy score (training): 0.864\n",
      "Accuracy score (validation): 0.860\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "\n",
    "lr_list = [0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1]\n",
    "\n",
    "for learning_rate in lr_list:\n",
    "    gb_clf = GradientBoostingClassifier(n_estimators=20, learning_rate=learning_rate, max_features=2, max_depth=2, random_state=0)\n",
    "    gb_clf.fit(x_train, y_train)\n",
    "\n",
    "    print(\"Learning rate: \", learning_rate)\n",
    "    print(\"Accuracy score (training): {0:.3f}\".format(gb_clf.score(x_train, y_train)))\n",
    "    print(\"Accuracy score (validation): {0:.3f}\".format(gb_clf.score(x_test, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8650131694468832"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gb_clf = GradientBoostingClassifier(n_estimators=100, max_features=2, max_depth=2, random_state=0)\n",
    "gb_clf.fit(x_train, y_train)\n",
    "gb_clf.score(x_test, y_test)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "gb_clf_df = thershold_matrix(gb_clf,x_test,y_test)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Threshold</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Recall</th>\n",
       "      <th>F1</th>\n",
       "      <th>TN</th>\n",
       "      <th>FP</th>\n",
       "      <th>FN</th>\n",
       "      <th>TP</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0.849868</td>\n",
       "      <td>0.36255</td>\n",
       "      <td>0.147967</td>\n",
       "      <td>0.82163</td>\n",
       "      <td>3781.0</td>\n",
       "      <td>160.0</td>\n",
       "      <td>524.0</td>\n",
       "      <td>91.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Threshold  Accuracy  Precision    Recall       F1      TN     FP     FN  \\\n",
       "3        0.2  0.849868    0.36255  0.147967  0.82163  3781.0  160.0  524.0   \n",
       "\n",
       "     TP  \n",
       "3  91.0  "
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gb_clf_df[gb_clf_df['F1']==gb_clf_df.F1.max()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Threshold</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Recall</th>\n",
       "      <th>F1</th>\n",
       "      <th>TN</th>\n",
       "      <th>FP</th>\n",
       "      <th>FN</th>\n",
       "      <th>TP</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.134987</td>\n",
       "      <td>0.134987</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.032109</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3941.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>615.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.10</td>\n",
       "      <td>0.251536</td>\n",
       "      <td>0.146828</td>\n",
       "      <td>0.944715</td>\n",
       "      <td>0.249608</td>\n",
       "      <td>565.0</td>\n",
       "      <td>3376.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>581.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.15</td>\n",
       "      <td>0.747366</td>\n",
       "      <td>0.247646</td>\n",
       "      <td>0.427642</td>\n",
       "      <td>0.773441</td>\n",
       "      <td>3142.0</td>\n",
       "      <td>799.0</td>\n",
       "      <td>352.0</td>\n",
       "      <td>263.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.849868</td>\n",
       "      <td>0.362550</td>\n",
       "      <td>0.147967</td>\n",
       "      <td>0.821630</td>\n",
       "      <td>3781.0</td>\n",
       "      <td>160.0</td>\n",
       "      <td>524.0</td>\n",
       "      <td>91.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.863257</td>\n",
       "      <td>0.423077</td>\n",
       "      <td>0.035772</td>\n",
       "      <td>0.810105</td>\n",
       "      <td>3911.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>593.0</td>\n",
       "      <td>22.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.30</td>\n",
       "      <td>0.864135</td>\n",
       "      <td>0.357143</td>\n",
       "      <td>0.008130</td>\n",
       "      <td>0.804040</td>\n",
       "      <td>3932.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>610.0</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>0.35</td>\n",
       "      <td>0.864794</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.001626</td>\n",
       "      <td>0.802718</td>\n",
       "      <td>3939.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>614.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0.40</td>\n",
       "      <td>0.865013</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.001626</td>\n",
       "      <td>0.802828</td>\n",
       "      <td>3940.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>614.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>0.45</td>\n",
       "      <td>0.865013</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.802405</td>\n",
       "      <td>3941.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>615.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.865013</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.802405</td>\n",
       "      <td>3941.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>615.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>0.55</td>\n",
       "      <td>0.865013</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.802405</td>\n",
       "      <td>3941.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>615.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>0.60</td>\n",
       "      <td>0.865013</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.802405</td>\n",
       "      <td>3941.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>615.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>0.65</td>\n",
       "      <td>0.865013</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.802405</td>\n",
       "      <td>3941.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>615.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>0.70</td>\n",
       "      <td>0.865013</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.802405</td>\n",
       "      <td>3941.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>615.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>0.75</td>\n",
       "      <td>0.865013</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.802405</td>\n",
       "      <td>3941.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>615.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>0.80</td>\n",
       "      <td>0.865013</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.802405</td>\n",
       "      <td>3941.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>615.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>16</td>\n",
       "      <td>0.85</td>\n",
       "      <td>0.865013</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.802405</td>\n",
       "      <td>3941.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>615.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>17</td>\n",
       "      <td>0.90</td>\n",
       "      <td>0.865013</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.802405</td>\n",
       "      <td>3941.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>615.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>18</td>\n",
       "      <td>0.95</td>\n",
       "      <td>0.865013</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.802405</td>\n",
       "      <td>3941.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>615.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>19</td>\n",
       "      <td>0.99</td>\n",
       "      <td>0.865013</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.802405</td>\n",
       "      <td>3941.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>615.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Threshold  Accuracy  Precision    Recall        F1      TN      FP     FN  \\\n",
       "0        0.05  0.134987   0.134987  1.000000  0.032109     0.0  3941.0    0.0   \n",
       "1        0.10  0.251536   0.146828  0.944715  0.249608   565.0  3376.0   34.0   \n",
       "2        0.15  0.747366   0.247646  0.427642  0.773441  3142.0   799.0  352.0   \n",
       "3        0.20  0.849868   0.362550  0.147967  0.821630  3781.0   160.0  524.0   \n",
       "4        0.25  0.863257   0.423077  0.035772  0.810105  3911.0    30.0  593.0   \n",
       "5        0.30  0.864135   0.357143  0.008130  0.804040  3932.0     9.0  610.0   \n",
       "6        0.35  0.864794   0.333333  0.001626  0.802718  3939.0     2.0  614.0   \n",
       "7        0.40  0.865013   0.500000  0.001626  0.802828  3940.0     1.0  614.0   \n",
       "8        0.45  0.865013   0.000000  0.000000  0.802405  3941.0     0.0  615.0   \n",
       "9        0.50  0.865013   0.000000  0.000000  0.802405  3941.0     0.0  615.0   \n",
       "10       0.55  0.865013   0.000000  0.000000  0.802405  3941.0     0.0  615.0   \n",
       "11       0.60  0.865013   0.000000  0.000000  0.802405  3941.0     0.0  615.0   \n",
       "12       0.65  0.865013   0.000000  0.000000  0.802405  3941.0     0.0  615.0   \n",
       "13       0.70  0.865013   0.000000  0.000000  0.802405  3941.0     0.0  615.0   \n",
       "14       0.75  0.865013   0.000000  0.000000  0.802405  3941.0     0.0  615.0   \n",
       "15       0.80  0.865013   0.000000  0.000000  0.802405  3941.0     0.0  615.0   \n",
       "16       0.85  0.865013   0.000000  0.000000  0.802405  3941.0     0.0  615.0   \n",
       "17       0.90  0.865013   0.000000  0.000000  0.802405  3941.0     0.0  615.0   \n",
       "18       0.95  0.865013   0.000000  0.000000  0.802405  3941.0     0.0  615.0   \n",
       "19       0.99  0.865013   0.000000  0.000000  0.802405  3941.0     0.0  615.0   \n",
       "\n",
       "       TP  \n",
       "0   615.0  \n",
       "1   581.0  \n",
       "2   263.0  \n",
       "3    91.0  \n",
       "4    22.0  \n",
       "5     5.0  \n",
       "6     1.0  \n",
       "7     1.0  \n",
       "8     0.0  \n",
       "9     0.0  \n",
       "10    0.0  \n",
       "11    0.0  \n",
       "12    0.0  \n",
       "13    0.0  \n",
       "14    0.0  \n",
       "15    0.0  \n",
       "16    0.0  \n",
       "17    0.0  \n",
       "18    0.0  \n",
       "19    0.0  "
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gb_clf_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_proba_df = pd.DataFrame(gb_clf.predict_proba(test_preprocessed))\n",
    "Y_test_pred = pred_proba_df.applymap(lambda x: 1 if x>0.15 else 0)\n",
    "y_pred = Y_test_pred.iloc[:,1]\n",
    "X = test.Col1\n",
    "df = pd.DataFrame({'Col1': X,'Col2': y_pred})\n",
    "df.to_csv('gb_clf.csv',index= False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<a href='gb_clf.csv' target='_blank'>gb_clf.csv</a><br>"
      ],
      "text/plain": [
       "/kaggle/working/gb_clf.csv"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython.display import FileLink\n",
    "\n",
    "FileLink(r'gb_clf.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "param_grid = {'C': [0.001, 0.01, 0.1, 1, 10, 100],'penalty' : ['l2'] }\n",
    "lgsReg_GridSearchCV = GridSearchCV(LogisticRegression(), param_grid,verbose=10000,n_jobs=2)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 6 candidates, totalling 18 fits\n",
      "[Parallel(n_jobs=2)]: Using backend LokyBackend with 2 concurrent workers.\n",
      "Pickling array (shape=(9605,), dtype=object).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Memmapping (shape=(628, 13665), dtype=float64) to new file /tmp/joblib_memmapping_folder_13_3516179738/13-139976999498080-d187aa6e62e3498482b73c4e9ff7b0af.pkl\n",
      "Memmapping (shape=(8977, 13665), dtype=uint8) to new file /tmp/joblib_memmapping_folder_13_3516179738/13-139976999498080-be4438bb693142019ca322cc2e5120a5.pkl\n",
      "Pickling array (shape=(628,), dtype=object).\n",
      "Pickling array (shape=(8977,), dtype=object).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Pickling array (shape=(9109,), dtype=int64).\n",
      "Pickling array (shape=(4556,), dtype=int64).\n",
      "Pickling array (shape=(9605,), dtype=object).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Memmapping (shape=(628, 13665), dtype=float64) to old file /tmp/joblib_memmapping_folder_13_3516179738/13-139976999498080-d187aa6e62e3498482b73c4e9ff7b0af.pkl\n",
      "Memmapping (shape=(8977, 13665), dtype=uint8) to old file /tmp/joblib_memmapping_folder_13_3516179738/13-139976999498080-be4438bb693142019ca322cc2e5120a5.pkl\n",
      "Pickling array (shape=(628,), dtype=object).\n",
      "Pickling array (shape=(8977,), dtype=object).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Pickling array (shape=(9110,), dtype=int64).\n",
      "Pickling array (shape=(4555,), dtype=int64).\n",
      "Pickling array (shape=(9605,), dtype=object).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Memmapping (shape=(628, 13665), dtype=float64) to old file /tmp/joblib_memmapping_folder_13_3516179738/13-139976999498080-d187aa6e62e3498482b73c4e9ff7b0af.pkl\n",
      "Memmapping (shape=(8977, 13665), dtype=uint8) to old file /tmp/joblib_memmapping_folder_13_3516179738/13-139976999498080-be4438bb693142019ca322cc2e5120a5.pkl\n",
      "Pickling array (shape=(628,), dtype=object).\n",
      "Pickling array (shape=(8977,), dtype=object).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Pickling array (shape=(9111,), dtype=int64).\n",
      "Pickling array (shape=(4554,), dtype=int64).\n",
      "[Parallel(n_jobs=2)]: Done   1 tasks      | elapsed:   13.1s\n",
      "Pickling array (shape=(9605,), dtype=object).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Memmapping (shape=(628, 13665), dtype=float64) to old file /tmp/joblib_memmapping_folder_13_3516179738/13-139976999498080-d187aa6e62e3498482b73c4e9ff7b0af.pkl\n",
      "Memmapping (shape=(8977, 13665), dtype=uint8) to old file /tmp/joblib_memmapping_folder_13_3516179738/13-139976999498080-be4438bb693142019ca322cc2e5120a5.pkl\n",
      "Pickling array (shape=(628,), dtype=object).\n",
      "Pickling array (shape=(8977,), dtype=object).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Pickling array (shape=(9109,), dtype=int64).\n",
      "Pickling array (shape=(4556,), dtype=int64).\n",
      "[Parallel(n_jobs=2)]: Done   2 tasks      | elapsed:   14.2s\n",
      "Pickling array (shape=(9605,), dtype=object).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Memmapping (shape=(628, 13665), dtype=float64) to old file /tmp/joblib_memmapping_folder_13_3516179738/13-139976999498080-d187aa6e62e3498482b73c4e9ff7b0af.pkl\n",
      "Memmapping (shape=(8977, 13665), dtype=uint8) to old file /tmp/joblib_memmapping_folder_13_3516179738/13-139976999498080-be4438bb693142019ca322cc2e5120a5.pkl\n",
      "Pickling array (shape=(628,), dtype=object).\n",
      "Pickling array (shape=(8977,), dtype=object).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Pickling array (shape=(9110,), dtype=int64).\n",
      "Pickling array (shape=(4555,), dtype=int64).\n",
      "[Parallel(n_jobs=2)]: Done   3 tasks      | elapsed:   23.8s\n",
      "Pickling array (shape=(9605,), dtype=object).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Memmapping (shape=(628, 13665), dtype=float64) to old file /tmp/joblib_memmapping_folder_13_3516179738/13-139976999498080-d187aa6e62e3498482b73c4e9ff7b0af.pkl\n",
      "Memmapping (shape=(8977, 13665), dtype=uint8) to old file /tmp/joblib_memmapping_folder_13_3516179738/13-139976999498080-be4438bb693142019ca322cc2e5120a5.pkl\n",
      "Pickling array (shape=(628,), dtype=object).\n",
      "Pickling array (shape=(8977,), dtype=object).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Pickling array (shape=(9111,), dtype=int64).\n",
      "Pickling array (shape=(4554,), dtype=int64).\n",
      "[Parallel(n_jobs=2)]: Done   4 tasks      | elapsed:   31.3s\n",
      "Pickling array (shape=(9605,), dtype=object).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Memmapping (shape=(628, 13665), dtype=float64) to old file /tmp/joblib_memmapping_folder_13_3516179738/13-139976999498080-d187aa6e62e3498482b73c4e9ff7b0af.pkl\n",
      "Memmapping (shape=(8977, 13665), dtype=uint8) to old file /tmp/joblib_memmapping_folder_13_3516179738/13-139976999498080-be4438bb693142019ca322cc2e5120a5.pkl\n",
      "Pickling array (shape=(628,), dtype=object).\n",
      "Pickling array (shape=(8977,), dtype=object).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Pickling array (shape=(9109,), dtype=int64).\n",
      "Pickling array (shape=(4556,), dtype=int64).\n",
      "[Parallel(n_jobs=2)]: Done   5 tasks      | elapsed:   43.5s\n",
      "Pickling array (shape=(9605,), dtype=object).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Memmapping (shape=(628, 13665), dtype=float64) to old file /tmp/joblib_memmapping_folder_13_3516179738/13-139976999498080-d187aa6e62e3498482b73c4e9ff7b0af.pkl\n",
      "Memmapping (shape=(8977, 13665), dtype=uint8) to old file /tmp/joblib_memmapping_folder_13_3516179738/13-139976999498080-be4438bb693142019ca322cc2e5120a5.pkl\n",
      "Pickling array (shape=(628,), dtype=object).\n",
      "Pickling array (shape=(8977,), dtype=object).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Pickling array (shape=(9110,), dtype=int64).\n",
      "Pickling array (shape=(4555,), dtype=int64).\n",
      "[Parallel(n_jobs=2)]: Done   6 tasks      | elapsed:   48.7s\n",
      "Pickling array (shape=(9605,), dtype=object).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Memmapping (shape=(628, 13665), dtype=float64) to old file /tmp/joblib_memmapping_folder_13_3516179738/13-139976999498080-d187aa6e62e3498482b73c4e9ff7b0af.pkl\n",
      "Memmapping (shape=(8977, 13665), dtype=uint8) to old file /tmp/joblib_memmapping_folder_13_3516179738/13-139976999498080-be4438bb693142019ca322cc2e5120a5.pkl\n",
      "Pickling array (shape=(628,), dtype=object).\n",
      "Pickling array (shape=(8977,), dtype=object).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Pickling array (shape=(9111,), dtype=int64).\n",
      "Pickling array (shape=(4554,), dtype=int64).\n",
      "[Parallel(n_jobs=2)]: Done   7 tasks      | elapsed:  1.3min\n",
      "Pickling array (shape=(9605,), dtype=object).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Memmapping (shape=(628, 13665), dtype=float64) to old file /tmp/joblib_memmapping_folder_13_3516179738/13-139976999498080-d187aa6e62e3498482b73c4e9ff7b0af.pkl\n",
      "Memmapping (shape=(8977, 13665), dtype=uint8) to old file /tmp/joblib_memmapping_folder_13_3516179738/13-139976999498080-be4438bb693142019ca322cc2e5120a5.pkl\n",
      "Pickling array (shape=(628,), dtype=object).\n",
      "Pickling array (shape=(8977,), dtype=object).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Pickling array (shape=(9109,), dtype=int64).\n",
      "Pickling array (shape=(4556,), dtype=int64).\n",
      "[Parallel(n_jobs=2)]: Done   8 tasks      | elapsed:  1.4min\n",
      "Pickling array (shape=(9605,), dtype=object).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Memmapping (shape=(628, 13665), dtype=float64) to old file /tmp/joblib_memmapping_folder_13_3516179738/13-139976999498080-d187aa6e62e3498482b73c4e9ff7b0af.pkl\n",
      "Memmapping (shape=(8977, 13665), dtype=uint8) to old file /tmp/joblib_memmapping_folder_13_3516179738/13-139976999498080-be4438bb693142019ca322cc2e5120a5.pkl\n",
      "Pickling array (shape=(628,), dtype=object).\n",
      "Pickling array (shape=(8977,), dtype=object).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Pickling array (shape=(9110,), dtype=int64).\n",
      "Pickling array (shape=(4555,), dtype=int64).\n",
      "[Parallel(n_jobs=2)]: Done   9 tasks      | elapsed:  1.9min\n",
      "Pickling array (shape=(9605,), dtype=object).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Memmapping (shape=(628, 13665), dtype=float64) to old file /tmp/joblib_memmapping_folder_13_3516179738/13-139976999498080-d187aa6e62e3498482b73c4e9ff7b0af.pkl\n",
      "Memmapping (shape=(8977, 13665), dtype=uint8) to old file /tmp/joblib_memmapping_folder_13_3516179738/13-139976999498080-be4438bb693142019ca322cc2e5120a5.pkl\n",
      "Pickling array (shape=(628,), dtype=object).\n",
      "Pickling array (shape=(8977,), dtype=object).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Pickling array (shape=(9111,), dtype=int64).\n",
      "Pickling array (shape=(4554,), dtype=int64).\n",
      "[Parallel(n_jobs=2)]: Done  10 tasks      | elapsed:  2.6min\n",
      "Pickling array (shape=(9605,), dtype=object).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Memmapping (shape=(628, 13665), dtype=float64) to old file /tmp/joblib_memmapping_folder_13_3516179738/13-139976999498080-d187aa6e62e3498482b73c4e9ff7b0af.pkl\n",
      "Memmapping (shape=(8977, 13665), dtype=uint8) to old file /tmp/joblib_memmapping_folder_13_3516179738/13-139976999498080-be4438bb693142019ca322cc2e5120a5.pkl\n",
      "Pickling array (shape=(628,), dtype=object).\n",
      "Pickling array (shape=(8977,), dtype=object).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Pickling array (shape=(9109,), dtype=int64).\n",
      "Pickling array (shape=(4556,), dtype=int64).\n",
      "[Parallel(n_jobs=2)]: Done  11 tasks      | elapsed:  3.1min\n",
      "Pickling array (shape=(9605,), dtype=object).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Memmapping (shape=(628, 13665), dtype=float64) to old file /tmp/joblib_memmapping_folder_13_3516179738/13-139976999498080-d187aa6e62e3498482b73c4e9ff7b0af.pkl\n",
      "Memmapping (shape=(8977, 13665), dtype=uint8) to old file /tmp/joblib_memmapping_folder_13_3516179738/13-139976999498080-be4438bb693142019ca322cc2e5120a5.pkl\n",
      "Pickling array (shape=(628,), dtype=object).\n",
      "Pickling array (shape=(8977,), dtype=object).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Pickling array (shape=(9110,), dtype=int64).\n",
      "Pickling array (shape=(4555,), dtype=int64).\n",
      "[Parallel(n_jobs=2)]: Done  12 tasks      | elapsed:  3.8min\n",
      "Pickling array (shape=(9605,), dtype=object).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Memmapping (shape=(628, 13665), dtype=float64) to old file /tmp/joblib_memmapping_folder_13_3516179738/13-139976999498080-d187aa6e62e3498482b73c4e9ff7b0af.pkl\n",
      "Memmapping (shape=(8977, 13665), dtype=uint8) to old file /tmp/joblib_memmapping_folder_13_3516179738/13-139976999498080-be4438bb693142019ca322cc2e5120a5.pkl\n",
      "Pickling array (shape=(628,), dtype=object).\n",
      "Pickling array (shape=(8977,), dtype=object).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Pickling array (shape=(9111,), dtype=int64).\n",
      "Pickling array (shape=(4554,), dtype=int64).\n",
      "[Parallel(n_jobs=2)]: Done  13 tasks      | elapsed:  6.0min\n",
      "Pickling array (shape=(9605,), dtype=object).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Memmapping (shape=(628, 13665), dtype=float64) to old file /tmp/joblib_memmapping_folder_13_3516179738/13-139976999498080-d187aa6e62e3498482b73c4e9ff7b0af.pkl\n",
      "Memmapping (shape=(8977, 13665), dtype=uint8) to old file /tmp/joblib_memmapping_folder_13_3516179738/13-139976999498080-be4438bb693142019ca322cc2e5120a5.pkl\n",
      "Pickling array (shape=(628,), dtype=object).\n",
      "Pickling array (shape=(8977,), dtype=object).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Pickling array (shape=(9109,), dtype=int64).\n",
      "Pickling array (shape=(4556,), dtype=int64).\n",
      "[Parallel(n_jobs=2)]: Done  14 tasks      | elapsed:  6.6min\n",
      "Pickling array (shape=(9605,), dtype=object).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Memmapping (shape=(628, 13665), dtype=float64) to old file /tmp/joblib_memmapping_folder_13_3516179738/13-139976999498080-d187aa6e62e3498482b73c4e9ff7b0af.pkl\n",
      "Memmapping (shape=(8977, 13665), dtype=uint8) to old file /tmp/joblib_memmapping_folder_13_3516179738/13-139976999498080-be4438bb693142019ca322cc2e5120a5.pkl\n",
      "Pickling array (shape=(628,), dtype=object).\n",
      "Pickling array (shape=(8977,), dtype=object).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Pickling array (shape=(9110,), dtype=int64).\n",
      "Pickling array (shape=(4555,), dtype=int64).\n",
      "[Parallel(n_jobs=2)]: Done  15 tasks      | elapsed:  8.7min\n",
      "Pickling array (shape=(9605,), dtype=object).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Memmapping (shape=(628, 13665), dtype=float64) to old file /tmp/joblib_memmapping_folder_13_3516179738/13-139976999498080-d187aa6e62e3498482b73c4e9ff7b0af.pkl\n",
      "Memmapping (shape=(8977, 13665), dtype=uint8) to old file /tmp/joblib_memmapping_folder_13_3516179738/13-139976999498080-be4438bb693142019ca322cc2e5120a5.pkl\n",
      "Pickling array (shape=(628,), dtype=object).\n",
      "Pickling array (shape=(8977,), dtype=object).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Pickling array (shape=(13665,), dtype=int64).\n",
      "Pickling array (shape=(9111,), dtype=int64).\n",
      "Pickling array (shape=(4554,), dtype=int64).\n",
      "[Parallel(n_jobs=2)]: Done  16 out of  18 | elapsed: 12.7min remaining:  1.6min\n",
      "[Parallel(n_jobs=2)]: Done  18 out of  18 | elapsed: 18.7min remaining:    0.0s\n",
      "[Parallel(n_jobs=2)]: Done  18 out of  18 | elapsed: 18.7min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv='warn', error_score='raise-deprecating',\n",
       "             estimator=LogisticRegression(C=1.0, class_weight=None, dual=False,\n",
       "                                          fit_intercept=True,\n",
       "                                          intercept_scaling=1, l1_ratio=None,\n",
       "                                          max_iter=100, multi_class='warn',\n",
       "                                          n_jobs=None, penalty='l2',\n",
       "                                          random_state=None, solver='warn',\n",
       "                                          tol=0.0001, verbose=0,\n",
       "                                          warm_start=False),\n",
       "             iid='warn', n_jobs=2,\n",
       "             param_grid={'C': [0.001, 0.01, 0.1, 1, 10, 100],\n",
       "                         'penalty': ['l2']},\n",
       "             pre_dispatch='2*n_jobs', refit=True, return_train_score=False,\n",
       "             scoring=None, verbose=10000)"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lgsReg_GridSearchCV.fit(x_train,y_train)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8665496049165935"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lgsReg_GridSearchCV.score(x_test,y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "lgsReg_GridSearchCV_df = thershold_matrix(lgsReg_GridSearchCV,x_test,y_test)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Threshold</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Recall</th>\n",
       "      <th>F1</th>\n",
       "      <th>TN</th>\n",
       "      <th>FP</th>\n",
       "      <th>FN</th>\n",
       "      <th>TP</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0.4</td>\n",
       "      <td>0.865233</td>\n",
       "      <td>0.501992</td>\n",
       "      <td>0.204878</td>\n",
       "      <td>0.839884</td>\n",
       "      <td>3816.0</td>\n",
       "      <td>125.0</td>\n",
       "      <td>489.0</td>\n",
       "      <td>126.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Threshold  Accuracy  Precision    Recall        F1      TN     FP     FN  \\\n",
       "7        0.4  0.865233   0.501992  0.204878  0.839884  3816.0  125.0  489.0   \n",
       "\n",
       "      TP  \n",
       "7  126.0  "
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lgsReg_GridSearchCV_df[lgsReg_GridSearchCV_df['F1']==lgsReg_GridSearchCV_df.F1.max()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = lgsReg_GridSearchCV.predict(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.825986464368194"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics.f1_score(y_test,y_pred,average = 'weighted')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_proba_df = pd.DataFrame(lgsReg_GridSearchCV.predict_proba(test_preprocessed))\n",
    "Y_test_pred = pred_proba_df.applymap(lambda x: 1 if x>0.25 else 0)\n",
    "y_pred = Y_test_pred.iloc[:,1]\n",
    "X = test.Col1\n",
    "df = pd.DataFrame({'Col1': X,'Col2': y_pred})\n",
    "df.to_csv('lgsReg_GridSearchCV_85.csv',index= False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LGBMClassifier(boosting_type='gbdt', class_weight=None, colsample_bytree=1.0,\n",
       "               importance_type='split', learning_rate=0.1, max_depth=-1,\n",
       "               min_child_samples=20, min_child_weight=0.001, min_split_gain=0.0,\n",
       "               n_estimators=100, n_jobs=-1, num_leaves=31, objective=None,\n",
       "               random_state=None, reg_alpha=0.0, reg_lambda=0.0, silent=True,\n",
       "               subsample=1.0, subsample_for_bin=200000, subsample_freq=0)"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import lightgbm as lgb\n",
    "model = lgb.LGBMClassifier()\n",
    "\n",
    "# fit the model with the training data\n",
    "model.fit(x_train,y_train)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}