Conoco Competition5.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Predicting equipment failures\n",
    "\n",
    "### Table 46\n",
    "\n",
    "### Anton Leontyev, Madi Muse, Sebastian Monroe"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data import and cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os \n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['classifier.json', 'classifier.h5', 'Conoco Competition3.ipynb', '.DS_Store', 'Conoco Competition2.ipynb', 'submission.csv', 'equip_failures_training_set.csv', 'submission1.csv', 'Conoco Competition4.ipynb', 'submission2.csv', 'Conoco Competition.ipynb', 'pickled_decision_tree.pkl', 'classifier.hdf5', 'Conoco Competition3.pdf', 'equip_failures_test_set.csv', '.ipynb_checkpoints']\n"
     ]
    }
   ],
   "source": [
    "arr = os.listdir()\n",
    "print(arr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = pd.read_csv('equip_failures_training_set.csv')\n",
    "df_test = pd.read_csv('equip_failures_test_set.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's inspect the dataset:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>target</th>\n",
       "      <th>sensor1_measure</th>\n",
       "      <th>sensor2_measure</th>\n",
       "      <th>sensor3_measure</th>\n",
       "      <th>sensor4_measure</th>\n",
       "      <th>sensor5_measure</th>\n",
       "      <th>sensor6_measure</th>\n",
       "      <th>sensor7_histogram_bin0</th>\n",
       "      <th>sensor7_histogram_bin1</th>\n",
       "      <th>...</th>\n",
       "      <th>sensor105_histogram_bin2</th>\n",
       "      <th>sensor105_histogram_bin3</th>\n",
       "      <th>sensor105_histogram_bin4</th>\n",
       "      <th>sensor105_histogram_bin5</th>\n",
       "      <th>sensor105_histogram_bin6</th>\n",
       "      <th>sensor105_histogram_bin7</th>\n",
       "      <th>sensor105_histogram_bin8</th>\n",
       "      <th>sensor105_histogram_bin9</th>\n",
       "      <th>sensor106_measure</th>\n",
       "      <th>sensor107_measure</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>76698</td>\n",
       "      <td>na</td>\n",
       "      <td>2130706438</td>\n",
       "      <td>280</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1240520</td>\n",
       "      <td>493384</td>\n",
       "      <td>721044</td>\n",
       "      <td>469792</td>\n",
       "      <td>339156</td>\n",
       "      <td>157956</td>\n",
       "      <td>73224</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>33058</td>\n",
       "      <td>na</td>\n",
       "      <td>0</td>\n",
       "      <td>na</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>421400</td>\n",
       "      <td>178064</td>\n",
       "      <td>293306</td>\n",
       "      <td>245416</td>\n",
       "      <td>133654</td>\n",
       "      <td>81140</td>\n",
       "      <td>97576</td>\n",
       "      <td>1500</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>41040</td>\n",
       "      <td>na</td>\n",
       "      <td>228</td>\n",
       "      <td>100</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>277378</td>\n",
       "      <td>159812</td>\n",
       "      <td>423992</td>\n",
       "      <td>409564</td>\n",
       "      <td>320746</td>\n",
       "      <td>158022</td>\n",
       "      <td>95128</td>\n",
       "      <td>514</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>70</td>\n",
       "      <td>66</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>240</td>\n",
       "      <td>46</td>\n",
       "      <td>58</td>\n",
       "      <td>44</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>60874</td>\n",
       "      <td>na</td>\n",
       "      <td>1368</td>\n",
       "      <td>458</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>622012</td>\n",
       "      <td>229790</td>\n",
       "      <td>405298</td>\n",
       "      <td>347188</td>\n",
       "      <td>286954</td>\n",
       "      <td>311560</td>\n",
       "      <td>433954</td>\n",
       "      <td>1218</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>59995</td>\n",
       "      <td>59996</td>\n",
       "      <td>0</td>\n",
       "      <td>153002</td>\n",
       "      <td>na</td>\n",
       "      <td>664</td>\n",
       "      <td>186</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>998500</td>\n",
       "      <td>566884</td>\n",
       "      <td>1290398</td>\n",
       "      <td>1218244</td>\n",
       "      <td>1019768</td>\n",
       "      <td>717762</td>\n",
       "      <td>898642</td>\n",
       "      <td>28588</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>59996</td>\n",
       "      <td>59997</td>\n",
       "      <td>0</td>\n",
       "      <td>2286</td>\n",
       "      <td>na</td>\n",
       "      <td>2130706538</td>\n",
       "      <td>224</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>10578</td>\n",
       "      <td>6760</td>\n",
       "      <td>21126</td>\n",
       "      <td>68424</td>\n",
       "      <td>136</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>59997</td>\n",
       "      <td>59998</td>\n",
       "      <td>0</td>\n",
       "      <td>112</td>\n",
       "      <td>0</td>\n",
       "      <td>2130706432</td>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>792</td>\n",
       "      <td>386</td>\n",
       "      <td>452</td>\n",
       "      <td>144</td>\n",
       "      <td>146</td>\n",
       "      <td>2622</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>59998</td>\n",
       "      <td>59999</td>\n",
       "      <td>0</td>\n",
       "      <td>80292</td>\n",
       "      <td>na</td>\n",
       "      <td>2130706432</td>\n",
       "      <td>494</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>699352</td>\n",
       "      <td>222654</td>\n",
       "      <td>347378</td>\n",
       "      <td>225724</td>\n",
       "      <td>194440</td>\n",
       "      <td>165070</td>\n",
       "      <td>802280</td>\n",
       "      <td>388422</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>59999</td>\n",
       "      <td>60000</td>\n",
       "      <td>0</td>\n",
       "      <td>40222</td>\n",
       "      <td>na</td>\n",
       "      <td>698</td>\n",
       "      <td>628</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>440066</td>\n",
       "      <td>183200</td>\n",
       "      <td>344546</td>\n",
       "      <td>254068</td>\n",
       "      <td>225148</td>\n",
       "      <td>158304</td>\n",
       "      <td>170384</td>\n",
       "      <td>158</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>60000 rows × 172 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          id  target  sensor1_measure sensor2_measure sensor3_measure  \\\n",
       "0          1       0            76698              na      2130706438   \n",
       "1          2       0            33058              na               0   \n",
       "2          3       0            41040              na             228   \n",
       "3          4       0               12               0              70   \n",
       "4          5       0            60874              na            1368   \n",
       "...      ...     ...              ...             ...             ...   \n",
       "59995  59996       0           153002              na             664   \n",
       "59996  59997       0             2286              na      2130706538   \n",
       "59997  59998       0              112               0      2130706432   \n",
       "59998  59999       0            80292              na      2130706432   \n",
       "59999  60000       0            40222              na             698   \n",
       "\n",
       "      sensor4_measure sensor5_measure sensor6_measure sensor7_histogram_bin0  \\\n",
       "0                 280               0               0                      0   \n",
       "1                  na               0               0                      0   \n",
       "2                 100               0               0                      0   \n",
       "3                  66               0              10                      0   \n",
       "4                 458               0               0                      0   \n",
       "...               ...             ...             ...                    ...   \n",
       "59995             186               0               0                      0   \n",
       "59996             224               0               0                      0   \n",
       "59997              18               0               0                      0   \n",
       "59998             494               0               0                      0   \n",
       "59999             628               0               0                      0   \n",
       "\n",
       "      sensor7_histogram_bin1  ... sensor105_histogram_bin2  \\\n",
       "0                          0  ...                  1240520   \n",
       "1                          0  ...                   421400   \n",
       "2                          0  ...                   277378   \n",
       "3                          0  ...                      240   \n",
       "4                          0  ...                   622012   \n",
       "...                      ...  ...                      ...   \n",
       "59995                      0  ...                   998500   \n",
       "59996                      0  ...                    10578   \n",
       "59997                      0  ...                      792   \n",
       "59998                      0  ...                   699352   \n",
       "59999                      0  ...                   440066   \n",
       "\n",
       "      sensor105_histogram_bin3 sensor105_histogram_bin4  \\\n",
       "0                       493384                   721044   \n",
       "1                       178064                   293306   \n",
       "2                       159812                   423992   \n",
       "3                           46                       58   \n",
       "4                       229790                   405298   \n",
       "...                        ...                      ...   \n",
       "59995                   566884                  1290398   \n",
       "59996                     6760                    21126   \n",
       "59997                      386                      452   \n",
       "59998                   222654                   347378   \n",
       "59999                   183200                   344546   \n",
       "\n",
       "      sensor105_histogram_bin5 sensor105_histogram_bin6  \\\n",
       "0                       469792                   339156   \n",
       "1                       245416                   133654   \n",
       "2                       409564                   320746   \n",
       "3                           44                       10   \n",
       "4                       347188                   286954   \n",
       "...                        ...                      ...   \n",
       "59995                  1218244                  1019768   \n",
       "59996                    68424                      136   \n",
       "59997                      144                      146   \n",
       "59998                   225724                   194440   \n",
       "59999                   254068                   225148   \n",
       "\n",
       "      sensor105_histogram_bin7 sensor105_histogram_bin8  \\\n",
       "0                       157956                    73224   \n",
       "1                        81140                    97576   \n",
       "2                       158022                    95128   \n",
       "3                            0                        0   \n",
       "4                       311560                   433954   \n",
       "...                        ...                      ...   \n",
       "59995                   717762                   898642   \n",
       "59996                        0                        0   \n",
       "59997                     2622                        0   \n",
       "59998                   165070                   802280   \n",
       "59999                   158304                   170384   \n",
       "\n",
       "      sensor105_histogram_bin9 sensor106_measure sensor107_measure  \n",
       "0                            0                 0                 0  \n",
       "1                         1500                 0                 0  \n",
       "2                          514                 0                 0  \n",
       "3                            0                 4                32  \n",
       "4                         1218                 0                 0  \n",
       "...                        ...               ...               ...  \n",
       "59995                    28588                 0                 0  \n",
       "59996                        0                 0                 0  \n",
       "59997                        0                 0                 0  \n",
       "59998                   388422                 0                 0  \n",
       "59999                      158                 0                 0  \n",
       "\n",
       "[60000 rows x 172 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's see what columns do we have:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['id',\n",
       " 'target',\n",
       " 'sensor1_measure',\n",
       " 'sensor2_measure',\n",
       " 'sensor3_measure',\n",
       " 'sensor4_measure',\n",
       " 'sensor5_measure',\n",
       " 'sensor6_measure',\n",
       " 'sensor7_histogram_bin0',\n",
       " 'sensor7_histogram_bin1',\n",
       " 'sensor7_histogram_bin2',\n",
       " 'sensor7_histogram_bin3',\n",
       " 'sensor7_histogram_bin4',\n",
       " 'sensor7_histogram_bin5',\n",
       " 'sensor7_histogram_bin6',\n",
       " 'sensor7_histogram_bin7',\n",
       " 'sensor7_histogram_bin8',\n",
       " 'sensor7_histogram_bin9',\n",
       " 'sensor8_measure',\n",
       " 'sensor9_measure',\n",
       " 'sensor10_measure',\n",
       " 'sensor11_measure',\n",
       " 'sensor12_measure',\n",
       " 'sensor13_measure',\n",
       " 'sensor14_measure',\n",
       " 'sensor15_measure',\n",
       " 'sensor16_measure',\n",
       " 'sensor17_measure',\n",
       " 'sensor18_measure',\n",
       " 'sensor19_measure',\n",
       " 'sensor20_measure',\n",
       " 'sensor21_measure',\n",
       " 'sensor22_measure',\n",
       " 'sensor23_measure',\n",
       " 'sensor24_histogram_bin0',\n",
       " 'sensor24_histogram_bin1',\n",
       " 'sensor24_histogram_bin2',\n",
       " 'sensor24_histogram_bin3',\n",
       " 'sensor24_histogram_bin4',\n",
       " 'sensor24_histogram_bin5',\n",
       " 'sensor24_histogram_bin6',\n",
       " 'sensor24_histogram_bin7',\n",
       " 'sensor24_histogram_bin8',\n",
       " 'sensor24_histogram_bin9',\n",
       " 'sensor25_histogram_bin0',\n",
       " 'sensor25_histogram_bin1',\n",
       " 'sensor25_histogram_bin2',\n",
       " 'sensor25_histogram_bin3',\n",
       " 'sensor25_histogram_bin4',\n",
       " 'sensor25_histogram_bin5',\n",
       " 'sensor25_histogram_bin6',\n",
       " 'sensor25_histogram_bin7',\n",
       " 'sensor25_histogram_bin8',\n",
       " 'sensor25_histogram_bin9',\n",
       " 'sensor26_histogram_bin0',\n",
       " 'sensor26_histogram_bin1',\n",
       " 'sensor26_histogram_bin2',\n",
       " 'sensor26_histogram_bin3',\n",
       " 'sensor26_histogram_bin4',\n",
       " 'sensor26_histogram_bin5',\n",
       " 'sensor26_histogram_bin6',\n",
       " 'sensor26_histogram_bin7',\n",
       " 'sensor26_histogram_bin8',\n",
       " 'sensor26_histogram_bin9',\n",
       " 'sensor27_measure',\n",
       " 'sensor28_measure',\n",
       " 'sensor29_measure',\n",
       " 'sensor30_measure',\n",
       " 'sensor31_measure',\n",
       " 'sensor32_measure',\n",
       " 'sensor33_measure',\n",
       " 'sensor34_measure',\n",
       " 'sensor35_measure',\n",
       " 'sensor36_measure',\n",
       " 'sensor37_measure',\n",
       " 'sensor38_measure',\n",
       " 'sensor39_measure',\n",
       " 'sensor40_measure',\n",
       " 'sensor41_measure',\n",
       " 'sensor42_measure',\n",
       " 'sensor43_measure',\n",
       " 'sensor44_measure',\n",
       " 'sensor45_measure',\n",
       " 'sensor46_measure',\n",
       " 'sensor47_measure',\n",
       " 'sensor48_measure',\n",
       " 'sensor49_measure',\n",
       " 'sensor50_measure',\n",
       " 'sensor51_measure',\n",
       " 'sensor52_measure',\n",
       " 'sensor53_measure',\n",
       " 'sensor54_measure',\n",
       " 'sensor55_measure',\n",
       " 'sensor56_measure',\n",
       " 'sensor57_measure',\n",
       " 'sensor58_measure',\n",
       " 'sensor59_measure',\n",
       " 'sensor60_measure',\n",
       " 'sensor61_measure',\n",
       " 'sensor62_measure',\n",
       " 'sensor63_measure',\n",
       " 'sensor64_histogram_bin0',\n",
       " 'sensor64_histogram_bin1',\n",
       " 'sensor64_histogram_bin2',\n",
       " 'sensor64_histogram_bin3',\n",
       " 'sensor64_histogram_bin4',\n",
       " 'sensor64_histogram_bin5',\n",
       " 'sensor64_histogram_bin6',\n",
       " 'sensor64_histogram_bin7',\n",
       " 'sensor64_histogram_bin8',\n",
       " 'sensor64_histogram_bin9',\n",
       " 'sensor65_measure',\n",
       " 'sensor66_measure',\n",
       " 'sensor67_measure',\n",
       " 'sensor68_measure',\n",
       " 'sensor69_histogram_bin0',\n",
       " 'sensor69_histogram_bin1',\n",
       " 'sensor69_histogram_bin2',\n",
       " 'sensor69_histogram_bin3',\n",
       " 'sensor69_histogram_bin4',\n",
       " 'sensor69_histogram_bin5',\n",
       " 'sensor69_histogram_bin6',\n",
       " 'sensor69_histogram_bin7',\n",
       " 'sensor69_histogram_bin8',\n",
       " 'sensor69_histogram_bin9',\n",
       " 'sensor70_measure',\n",
       " 'sensor71_measure',\n",
       " 'sensor72_measure',\n",
       " 'sensor73_measure',\n",
       " 'sensor74_measure',\n",
       " 'sensor75_measure',\n",
       " 'sensor76_measure',\n",
       " 'sensor77_measure',\n",
       " 'sensor78_measure',\n",
       " 'sensor79_measure',\n",
       " 'sensor80_measure',\n",
       " 'sensor81_measure',\n",
       " 'sensor82_measure',\n",
       " 'sensor83_measure',\n",
       " 'sensor84_measure',\n",
       " 'sensor85_measure',\n",
       " 'sensor86_measure',\n",
       " 'sensor87_measure',\n",
       " 'sensor88_measure',\n",
       " 'sensor89_measure',\n",
       " 'sensor90_measure',\n",
       " 'sensor91_measure',\n",
       " 'sensor92_measure',\n",
       " 'sensor93_measure',\n",
       " 'sensor94_measure',\n",
       " 'sensor95_measure',\n",
       " 'sensor96_measure',\n",
       " 'sensor97_measure',\n",
       " 'sensor98_measure',\n",
       " 'sensor99_measure',\n",
       " 'sensor100_measure',\n",
       " 'sensor101_measure',\n",
       " 'sensor102_measure',\n",
       " 'sensor103_measure',\n",
       " 'sensor104_measure',\n",
       " 'sensor105_histogram_bin0',\n",
       " 'sensor105_histogram_bin1',\n",
       " 'sensor105_histogram_bin2',\n",
       " 'sensor105_histogram_bin3',\n",
       " 'sensor105_histogram_bin4',\n",
       " 'sensor105_histogram_bin5',\n",
       " 'sensor105_histogram_bin6',\n",
       " 'sensor105_histogram_bin7',\n",
       " 'sensor105_histogram_bin8',\n",
       " 'sensor105_histogram_bin9',\n",
       " 'sensor106_measure',\n",
       " 'sensor107_measure']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(df_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "What types are these columns?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id                           int64\n",
       "target                       int64\n",
       "sensor1_measure              int64\n",
       "sensor2_measure             object\n",
       "sensor3_measure             object\n",
       "                             ...  \n",
       "sensor105_histogram_bin7    object\n",
       "sensor105_histogram_bin8    object\n",
       "sensor105_histogram_bin9    object\n",
       "sensor106_measure           object\n",
       "sensor107_measure           object\n",
       "Length: 172, dtype: object"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.dtypes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Object means they need to be coerced to the numerical variables; \n",
    "They are listed as object because na are parsed as strings."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id                            int64\n",
       "target                        int64\n",
       "sensor1_measure               int64\n",
       "sensor2_measure             float64\n",
       "sensor3_measure             float64\n",
       "                             ...   \n",
       "sensor105_histogram_bin7    float64\n",
       "sensor105_histogram_bin8    float64\n",
       "sensor105_histogram_bin9    float64\n",
       "sensor106_measure           float64\n",
       "sensor107_measure           float64\n",
       "Length: 172, dtype: object"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train1 = df_train.apply(pd.to_numeric, errors='coerce')\n",
    "df_train1.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>target</th>\n",
       "      <th>sensor1_measure</th>\n",
       "      <th>sensor2_measure</th>\n",
       "      <th>sensor3_measure</th>\n",
       "      <th>sensor4_measure</th>\n",
       "      <th>sensor5_measure</th>\n",
       "      <th>sensor6_measure</th>\n",
       "      <th>sensor7_histogram_bin0</th>\n",
       "      <th>sensor7_histogram_bin1</th>\n",
       "      <th>...</th>\n",
       "      <th>sensor105_histogram_bin2</th>\n",
       "      <th>sensor105_histogram_bin3</th>\n",
       "      <th>sensor105_histogram_bin4</th>\n",
       "      <th>sensor105_histogram_bin5</th>\n",
       "      <th>sensor105_histogram_bin6</th>\n",
       "      <th>sensor105_histogram_bin7</th>\n",
       "      <th>sensor105_histogram_bin8</th>\n",
       "      <th>sensor105_histogram_bin9</th>\n",
       "      <th>sensor106_measure</th>\n",
       "      <th>sensor107_measure</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>76698</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.130706e+09</td>\n",
       "      <td>280.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1240520.0</td>\n",
       "      <td>493384.0</td>\n",
       "      <td>721044.0</td>\n",
       "      <td>469792.0</td>\n",
       "      <td>339156.0</td>\n",
       "      <td>157956.0</td>\n",
       "      <td>73224.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>33058</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>421400.0</td>\n",
       "      <td>178064.0</td>\n",
       "      <td>293306.0</td>\n",
       "      <td>245416.0</td>\n",
       "      <td>133654.0</td>\n",
       "      <td>81140.0</td>\n",
       "      <td>97576.0</td>\n",
       "      <td>1500.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>41040</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.280000e+02</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>277378.0</td>\n",
       "      <td>159812.0</td>\n",
       "      <td>423992.0</td>\n",
       "      <td>409564.0</td>\n",
       "      <td>320746.0</td>\n",
       "      <td>158022.0</td>\n",
       "      <td>95128.0</td>\n",
       "      <td>514.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.000000e+01</td>\n",
       "      <td>66.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>240.0</td>\n",
       "      <td>46.0</td>\n",
       "      <td>58.0</td>\n",
       "      <td>44.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>32.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>60874</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.368000e+03</td>\n",
       "      <td>458.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>622012.0</td>\n",
       "      <td>229790.0</td>\n",
       "      <td>405298.0</td>\n",
       "      <td>347188.0</td>\n",
       "      <td>286954.0</td>\n",
       "      <td>311560.0</td>\n",
       "      <td>433954.0</td>\n",
       "      <td>1218.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>59995</td>\n",
       "      <td>59996</td>\n",
       "      <td>0</td>\n",
       "      <td>153002</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.640000e+02</td>\n",
       "      <td>186.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>998500.0</td>\n",
       "      <td>566884.0</td>\n",
       "      <td>1290398.0</td>\n",
       "      <td>1218244.0</td>\n",
       "      <td>1019768.0</td>\n",
       "      <td>717762.0</td>\n",
       "      <td>898642.0</td>\n",
       "      <td>28588.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>59996</td>\n",
       "      <td>59997</td>\n",
       "      <td>0</td>\n",
       "      <td>2286</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.130707e+09</td>\n",
       "      <td>224.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>10578.0</td>\n",
       "      <td>6760.0</td>\n",
       "      <td>21126.0</td>\n",
       "      <td>68424.0</td>\n",
       "      <td>136.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>59997</td>\n",
       "      <td>59998</td>\n",
       "      <td>0</td>\n",
       "      <td>112</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.130706e+09</td>\n",
       "      <td>18.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>792.0</td>\n",
       "      <td>386.0</td>\n",
       "      <td>452.0</td>\n",
       "      <td>144.0</td>\n",
       "      <td>146.0</td>\n",
       "      <td>2622.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>59998</td>\n",
       "      <td>59999</td>\n",
       "      <td>0</td>\n",
       "      <td>80292</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.130706e+09</td>\n",
       "      <td>494.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>699352.0</td>\n",
       "      <td>222654.0</td>\n",
       "      <td>347378.0</td>\n",
       "      <td>225724.0</td>\n",
       "      <td>194440.0</td>\n",
       "      <td>165070.0</td>\n",
       "      <td>802280.0</td>\n",
       "      <td>388422.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>59999</td>\n",
       "      <td>60000</td>\n",
       "      <td>0</td>\n",
       "      <td>40222</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.980000e+02</td>\n",
       "      <td>628.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>440066.0</td>\n",
       "      <td>183200.0</td>\n",
       "      <td>344546.0</td>\n",
       "      <td>254068.0</td>\n",
       "      <td>225148.0</td>\n",
       "      <td>158304.0</td>\n",
       "      <td>170384.0</td>\n",
       "      <td>158.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>60000 rows × 172 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          id  target  sensor1_measure  sensor2_measure  sensor3_measure  \\\n",
       "0          1       0            76698              0.0     2.130706e+09   \n",
       "1          2       0            33058              0.0     0.000000e+00   \n",
       "2          3       0            41040              0.0     2.280000e+02   \n",
       "3          4       0               12              0.0     7.000000e+01   \n",
       "4          5       0            60874              0.0     1.368000e+03   \n",
       "...      ...     ...              ...              ...              ...   \n",
       "59995  59996       0           153002              0.0     6.640000e+02   \n",
       "59996  59997       0             2286              0.0     2.130707e+09   \n",
       "59997  59998       0              112              0.0     2.130706e+09   \n",
       "59998  59999       0            80292              0.0     2.130706e+09   \n",
       "59999  60000       0            40222              0.0     6.980000e+02   \n",
       "\n",
       "       sensor4_measure  sensor5_measure  sensor6_measure  \\\n",
       "0                280.0              0.0              0.0   \n",
       "1                  0.0              0.0              0.0   \n",
       "2                100.0              0.0              0.0   \n",
       "3                 66.0              0.0             10.0   \n",
       "4                458.0              0.0              0.0   \n",
       "...                ...              ...              ...   \n",
       "59995            186.0              0.0              0.0   \n",
       "59996            224.0              0.0              0.0   \n",
       "59997             18.0              0.0              0.0   \n",
       "59998            494.0              0.0              0.0   \n",
       "59999            628.0              0.0              0.0   \n",
       "\n",
       "       sensor7_histogram_bin0  sensor7_histogram_bin1  ...  \\\n",
       "0                         0.0                     0.0  ...   \n",
       "1                         0.0                     0.0  ...   \n",
       "2                         0.0                     0.0  ...   \n",
       "3                         0.0                     0.0  ...   \n",
       "4                         0.0                     0.0  ...   \n",
       "...                       ...                     ...  ...   \n",
       "59995                     0.0                     0.0  ...   \n",
       "59996                     0.0                     0.0  ...   \n",
       "59997                     0.0                     0.0  ...   \n",
       "59998                     0.0                     0.0  ...   \n",
       "59999                     0.0                     0.0  ...   \n",
       "\n",
       "       sensor105_histogram_bin2  sensor105_histogram_bin3  \\\n",
       "0                     1240520.0                  493384.0   \n",
       "1                      421400.0                  178064.0   \n",
       "2                      277378.0                  159812.0   \n",
       "3                         240.0                      46.0   \n",
       "4                      622012.0                  229790.0   \n",
       "...                         ...                       ...   \n",
       "59995                  998500.0                  566884.0   \n",
       "59996                   10578.0                    6760.0   \n",
       "59997                     792.0                     386.0   \n",
       "59998                  699352.0                  222654.0   \n",
       "59999                  440066.0                  183200.0   \n",
       "\n",
       "       sensor105_histogram_bin4  sensor105_histogram_bin5  \\\n",
       "0                      721044.0                  469792.0   \n",
       "1                      293306.0                  245416.0   \n",
       "2                      423992.0                  409564.0   \n",
       "3                          58.0                      44.0   \n",
       "4                      405298.0                  347188.0   \n",
       "...                         ...                       ...   \n",
       "59995                 1290398.0                 1218244.0   \n",
       "59996                   21126.0                   68424.0   \n",
       "59997                     452.0                     144.0   \n",
       "59998                  347378.0                  225724.0   \n",
       "59999                  344546.0                  254068.0   \n",
       "\n",
       "       sensor105_histogram_bin6  sensor105_histogram_bin7  \\\n",
       "0                      339156.0                  157956.0   \n",
       "1                      133654.0                   81140.0   \n",
       "2                      320746.0                  158022.0   \n",
       "3                          10.0                       0.0   \n",
       "4                      286954.0                  311560.0   \n",
       "...                         ...                       ...   \n",
       "59995                 1019768.0                  717762.0   \n",
       "59996                     136.0                       0.0   \n",
       "59997                     146.0                    2622.0   \n",
       "59998                  194440.0                  165070.0   \n",
       "59999                  225148.0                  158304.0   \n",
       "\n",
       "       sensor105_histogram_bin8  sensor105_histogram_bin9  sensor106_measure  \\\n",
       "0                       73224.0                       0.0                0.0   \n",
       "1                       97576.0                    1500.0                0.0   \n",
       "2                       95128.0                     514.0                0.0   \n",
       "3                           0.0                       0.0                4.0   \n",
       "4                      433954.0                    1218.0                0.0   \n",
       "...                         ...                       ...                ...   \n",
       "59995                  898642.0                   28588.0                0.0   \n",
       "59996                       0.0                       0.0                0.0   \n",
       "59997                       0.0                       0.0                0.0   \n",
       "59998                  802280.0                  388422.0                0.0   \n",
       "59999                  170384.0                     158.0                0.0   \n",
       "\n",
       "       sensor107_measure  \n",
       "0                    0.0  \n",
       "1                    0.0  \n",
       "2                    0.0  \n",
       "3                   32.0  \n",
       "4                    0.0  \n",
       "...                  ...  \n",
       "59995                0.0  \n",
       "59996                0.0  \n",
       "59997                0.0  \n",
       "59998                0.0  \n",
       "59999                0.0  \n",
       "\n",
       "[60000 rows x 172 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train1.fillna(0) # we can consider no reading as reading with value 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, let's aggregate the continous sensor recordings. The simplest aggregation is averaging.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "sensor7 = ['sensor7_histogram_bin0',\n",
    " 'sensor7_histogram_bin1',\n",
    " 'sensor7_histogram_bin2',\n",
    " 'sensor7_histogram_bin3',\n",
    " 'sensor7_histogram_bin4',\n",
    " 'sensor7_histogram_bin5',\n",
    " 'sensor7_histogram_bin6',\n",
    " 'sensor7_histogram_bin7',\n",
    " 'sensor7_histogram_bin8',\n",
    " 'sensor7_histogram_bin9']\n",
    "sensor24 = ['sensor24_histogram_bin0',\n",
    " 'sensor24_histogram_bin1',\n",
    " 'sensor24_histogram_bin2',\n",
    " 'sensor24_histogram_bin3',\n",
    " 'sensor24_histogram_bin4',\n",
    " 'sensor24_histogram_bin5',\n",
    " 'sensor24_histogram_bin6',\n",
    " 'sensor24_histogram_bin7',\n",
    " 'sensor24_histogram_bin8',\n",
    " 'sensor24_histogram_bin9']\n",
    "sensor25 = ['sensor25_histogram_bin0',\n",
    " 'sensor25_histogram_bin1',\n",
    " 'sensor25_histogram_bin2',\n",
    " 'sensor25_histogram_bin3',\n",
    " 'sensor25_histogram_bin4',\n",
    " 'sensor25_histogram_bin5',\n",
    " 'sensor25_histogram_bin6',\n",
    " 'sensor25_histogram_bin7',\n",
    " 'sensor25_histogram_bin8',\n",
    " 'sensor25_histogram_bin9']\n",
    "sensor26 = ['sensor26_histogram_bin0',\n",
    " 'sensor26_histogram_bin1',\n",
    " 'sensor26_histogram_bin2',\n",
    " 'sensor26_histogram_bin3',\n",
    " 'sensor26_histogram_bin4',\n",
    " 'sensor26_histogram_bin5',\n",
    " 'sensor26_histogram_bin6',\n",
    " 'sensor26_histogram_bin7',\n",
    " 'sensor26_histogram_bin8',\n",
    " 'sensor26_histogram_bin9']\n",
    "sensor64 = ['sensor64_histogram_bin0',\n",
    " 'sensor64_histogram_bin1',\n",
    " 'sensor64_histogram_bin2',\n",
    " 'sensor64_histogram_bin3',\n",
    " 'sensor64_histogram_bin4',\n",
    " 'sensor64_histogram_bin5',\n",
    " 'sensor64_histogram_bin6',\n",
    " 'sensor64_histogram_bin7',\n",
    " 'sensor64_histogram_bin8',\n",
    " 'sensor64_histogram_bin9']\n",
    "sensor69 = ['sensor69_histogram_bin0',\n",
    " 'sensor69_histogram_bin1',\n",
    " 'sensor69_histogram_bin2',\n",
    " 'sensor69_histogram_bin3',\n",
    " 'sensor69_histogram_bin4',\n",
    " 'sensor69_histogram_bin5',\n",
    " 'sensor69_histogram_bin6',\n",
    " 'sensor69_histogram_bin7',\n",
    " 'sensor69_histogram_bin8',\n",
    " 'sensor69_histogram_bin9']\n",
    "sensor105 = ['sensor105_histogram_bin0',\n",
    " 'sensor105_histogram_bin1',\n",
    " 'sensor105_histogram_bin2',\n",
    " 'sensor105_histogram_bin3',\n",
    " 'sensor105_histogram_bin4',\n",
    " 'sensor105_histogram_bin5',\n",
    " 'sensor105_histogram_bin6',\n",
    " 'sensor105_histogram_bin7',\n",
    " 'sensor105_histogram_bin8',\n",
    " 'sensor105_histogram_bin9']\n",
    "df_train1['sensor7_average'] = df_train1[sensor7].mean(axis=1)\n",
    "df_train1['sensor24_average'] = df_train1[sensor24].mean(axis=1)\n",
    "df_train1['sensor25_average'] = df_train1[sensor25].mean(axis=1)\n",
    "df_train1['sensor26_average'] = df_train1[sensor26].mean(axis=1)\n",
    "df_train1['sensor64_average'] = df_train1[sensor64].mean(axis=1)\n",
    "df_train1['sensor69_average'] = df_train1[sensor69].mean(axis=1)\n",
    "df_train1['sensor105_average'] = df_train1[sensor105].mean(axis=1)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now that we created the averaged columns, let's simplify the data a little bit and get rid of histogram values. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "not_hists = [col for col in df_train1.columns if not  'histogram' in col]\n",
    "df_train_2 = df_train1[not_hists]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Are classes balanced in our dataset? Let's count how many instances of \"1\" do we have in our dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1000"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train_2['target'].astype(bool).sum(axis=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Transforming the data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Recall that we have 60000 rows total. Only 1000 instances of class 2 (underground failures, coded 1), while we have 59000 instances of class 1 (surface-related, coded 0). This is clearly a problem, because any model trained for predicting will achieve great accuracy. But this accuracy would actually be moot - the model will effectively be predicting one class.\n",
    "\n",
    "A techinque to somewhat combat it can be found in resampling. More accurately, we can \"upsample\" the underrepresented dataset by performing sampling with replacement. This technique is also known as bootstrapping. We will upsample the dataset to make it even."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    59000\n",
       "0    59000\n",
       "Name: target, dtype: int64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_majority = df_train_2[df_train_2.target==0]\n",
    "df_minority = df_train_2[df_train_2.target==1]\n",
    "from sklearn.utils import resample\n",
    "# Upsample minority class\n",
    "df_minority_upsampled = resample(df_minority, \n",
    "                                 replace=True,     # sample with replacement\n",
    "                                 n_samples=59000)    # to match majority class\n",
    "df_upsampled = pd.concat([df_majority, df_minority_upsampled])\n",
    "df_upsampled.target.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now the classes are balanced."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_upsampled = df_upsampled.fillna(0)\n",
    "y = df_upsampled.target\n",
    "X = df_upsampled.drop(['target','id'], axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, let's evaluate how different models fare on accuracy. We are going to use a cross-validation algorithm that splits the data into K parts (\"folds\") and iteratively treats 1 of these folds as validation and the rest as training. This process is repeated k times and aggregated measures is generated. We will select the model with the best accuracy and the lowest variability in accuracy (i.e., lowest difference in accuracy between folds)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import model_selection\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "#from sklearn.svm import SVC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/agleontiev/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/agleontiev/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/agleontiev/anaconda3/lib/python3.7/site-packages/sklearn/svm/base.py:929: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n",
      "/Users/agleontiev/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/agleontiev/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/agleontiev/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/agleontiev/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/agleontiev/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/agleontiev/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/agleontiev/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "/Users/agleontiev/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR: 0.941492 (0.015228)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/agleontiev/anaconda3/lib/python3.7/site-packages/sklearn/discriminant_analysis.py:388: UserWarning: Variables are collinear.\n",
      "  warnings.warn(\"Variables are collinear.\")\n",
      "/Users/agleontiev/anaconda3/lib/python3.7/site-packages/sklearn/discriminant_analysis.py:388: UserWarning: Variables are collinear.\n",
      "  warnings.warn(\"Variables are collinear.\")\n",
      "/Users/agleontiev/anaconda3/lib/python3.7/site-packages/sklearn/discriminant_analysis.py:388: UserWarning: Variables are collinear.\n",
      "  warnings.warn(\"Variables are collinear.\")\n",
      "/Users/agleontiev/anaconda3/lib/python3.7/site-packages/sklearn/discriminant_analysis.py:388: UserWarning: Variables are collinear.\n",
      "  warnings.warn(\"Variables are collinear.\")\n",
      "/Users/agleontiev/anaconda3/lib/python3.7/site-packages/sklearn/discriminant_analysis.py:388: UserWarning: Variables are collinear.\n",
      "  warnings.warn(\"Variables are collinear.\")\n",
      "/Users/agleontiev/anaconda3/lib/python3.7/site-packages/sklearn/discriminant_analysis.py:388: UserWarning: Variables are collinear.\n",
      "  warnings.warn(\"Variables are collinear.\")\n",
      "/Users/agleontiev/anaconda3/lib/python3.7/site-packages/sklearn/discriminant_analysis.py:388: UserWarning: Variables are collinear.\n",
      "  warnings.warn(\"Variables are collinear.\")\n",
      "/Users/agleontiev/anaconda3/lib/python3.7/site-packages/sklearn/discriminant_analysis.py:388: UserWarning: Variables are collinear.\n",
      "  warnings.warn(\"Variables are collinear.\")\n",
      "/Users/agleontiev/anaconda3/lib/python3.7/site-packages/sklearn/discriminant_analysis.py:388: UserWarning: Variables are collinear.\n",
      "  warnings.warn(\"Variables are collinear.\")\n",
      "/Users/agleontiev/anaconda3/lib/python3.7/site-packages/sklearn/discriminant_analysis.py:388: UserWarning: Variables are collinear.\n",
      "  warnings.warn(\"Variables are collinear.\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LDA: 0.903924 (0.048571)\n",
      "KNN: 0.990246 (0.009781)\n",
      "CART: 0.996754 (0.003309)\n",
      "NB: 0.925153 (0.039405)\n"
     ]
    }
   ],
   "source": [
    "models = []\n",
    "models.append(('LR', LogisticRegression()))\n",
    "models.append(('LDA', LinearDiscriminantAnalysis()))\n",
    "models.append(('KNN', KNeighborsClassifier()))\n",
    "models.append(('CART', DecisionTreeClassifier()))\n",
    "models.append(('NB', GaussianNB()))\n",
    "#models.append(('SVM', SVC()))\n",
    "results = []\n",
    "names = []\n",
    "scoring = 'accuracy'\n",
    "for name, model in models:\n",
    "\tkfold = model_selection.KFold(n_splits=10)\n",
    "\tcv_results = model_selection.cross_val_score(model, X, y, cv=kfold, scoring=scoring)\n",
    "\tresults.append(cv_results)\n",
    "\tnames.append(name)\n",
    "\tmsg = \"%s: %f (%f)\" % (name, cv_results.mean(), cv_results.std())\n",
    "\tprint(msg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's visualize the results of our model comparison, with y-axis representing accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEVCAYAAADuAi4fAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAbpElEQVR4nO3df7RU5X3v8fdHFE0E9CAnGgHBpJjrqSEaR0zTKkRXFbypRLmpEK0/rivk3lzTWxOTauoqR1JrV2oSm16SlLTGq6YSkmpKbm3RRTS/qi1DEAwiiKTKEY1HIVpjWkS/94+9T9gOc85sPHPODM/5vNaaxd77eWbPd+85fGbPs2f2KCIwM7N0HdDqAszMbGg56M3MEuegNzNLnIPezCxxDnozs8Q56M3MEuegt2RIulDSPa2uo4+kN0n6jqQXJH1zmB97lqSekn27Jd0+1DVZ6zjobS+SPiSpKuklSU9L+kdJv9XquhqJiK9HxFmtrqPgvwFHAkdExAdbXYyNXA56ex1JHwduAv6ULKSOAb4EzG1lXY1IOrDVNdQxBdgcEbtbXYiNbA56+xVJhwGLgf8VEXdGxC8i4pWI+E5EfDLvc7CkmyRtz283STo4b5slqUfSpyQ9m78b+ICkcyRtlrRD0qcLj9ct6VuSviHp3yX9WNK7Cu1XS3o8b3tE0nmFtksl/UjSFyTtALrzZT/M25W3PZsPnayXdELfdkq6VVKvpCckXSvpgMJ6fyjpRkk7Jf1U0pwB9tnxku6X9HNJGySdmy+/Dvhj4IL8ndHlde7bLembkm7Pt/FhScdJuiave5ukswr9j5a0It+PWyR9uND2Jkm35DU/ApxS81hHS/q7fJt/Kun3+9meQ/J6ns+3abWkI/vbfts/OOit6DeAQ4C7BujzR8B7gBOBdwEzgGsL7Ufl65hIFnRfBS4CTgZOA/5Y0tsK/ecC3wTGA38LfFvSQXnb4/l9DgOuA26X9NbCfU8FtgJvAa6vqfMs4HTgOOBw4ALg+bztL/N1vg2YCVwMXFaz3k3ABOCzwN9IUu2OyOv8DnBPXsPHgK9LekdELCJ7V/SNiBgTEX9Te//c7wC3AR3AWmAl2f/LiWQvun9V6HsH0AMcTTYs9KeSzszbFgFvz29nA5cU6jwgr3Ndvt4zgT+QdHadei7J981k4AjgfwC/7Kd2219EhG++EREAFwLPNOjzOHBOYf5s4N/y6VlkoTAqnx8LBHBqof8a4AP5dDfwYKHtAOBp4LR+HvshYG4+fSnwZE37pcAP8+kzgM1kL0oHFPqMAv4T6Cos+whwf2EdWwptb8634ag69ZwGPFOz/juA7sL23T7AvuwG7i3M/w7wUp39dzhZ8L4KjC30vwG4JZ/eCswutC0EevLpU+vsq2uAr9XWCfx34J+B6a3+e/SteTcf0VvR88CEBuPdRwNPFOafyJf9ah0R8Wo+3Xck+LNC+y+BMYX5bX0TEfEae45YkXSxpIfyIYSfAyeQHWXvdd9aEfFd4P8AS4CfSVoqaVx+/9F1tmFiYf6ZwnpezieLNfc5GtiW193fuhqp3TfP1dl/Y/LH2hER/97PYx3N6/dHcfumAEf37cd8X36a7BxMrdvI3lUsy4fmPlt4h2X7KQe9FT0A/AfwgQH6bCcLjj7H5MveqMl9E/kQwyRgu6QpZMM+V5B9auVw4CdAcQhlwEuvRsQXI+Jk4NfJhnA+CTwHvFJnG556A7VvByb3je8Pcl1lHmu8pLH9PNbTFPZl3tZnG/DTiDi8cBsbEefUPkhk52Sui4gu4L3A+8mGtmw/5qC3X4mIF8jG1ZfkJ1HfLOkgSXMkfTbvdgdwraROSRPy/oP5DPbJks7P30X8AdmwyoPAoWRB3gsg6TKyI/pSJJ0i6dT8aPQXZC9gr+ZHy8uB6yWNzV9QPv4Gt+Ff8nV/Kt9Ps8iGX5a9gXUNKCK2kQ2p3JCfMJ0OXA58Pe+yHLhGUoekSWTnC/r8K/CipD/MT9qOknSCpNedsAWQ9D5J75Q0CniR7EXx1dp+tn9x0NvrRMTnyYLvWrKQ3UZ2VP3tvMufAFVgPfAw8ON82Rv192QnSncCvwecnx9VPgJ8juxdxs+AdwI/2of1jiN7R7CTbBjjeeDGvO1jZAG9Ffgh2Ungm/e18IjYBZwLzCF7p/Al4OKIeHRf11XSAmAq2dH9XcCiiLg3b7uObDt/SnZy+LZCna+SvQCdmLc/B/w12UnXWkcB3yIL+Y3A9xjcC7m1AUX4h0esNSR1A78WERe1uhazlPmI3swscQ56M7PEeejGzCxxPqI3M0ucg97MLHEOejOzxDnozcwS56A3M0ucg97MLHEOejOzxDnozcwS56A3M0ucg97MLHEOejOzxDnozcwS56A3M0ucg97MLHEHtrqAWhMmTIipU6e2ugwzs/3KmjVrnouIznptbRf0U6dOpVqttroMM7P9iqQn+mvz0I2ZWeIc9GZmiXPQm5klzkFvZpY4B72ZWeIaBr2kmyU9K+kn/bRL0hclbZG0XtK7C22XSHosv13SzMLNzKycMkf0twCzB2ifA0zLbwuBLwNIGg8sAk4FZgCLJHUMplgzM9t3DYM+Ir4P7Bigy1zg1sg8CBwu6a3A2cC9EbEjInYC9zLwC4aZmQ2BZozRTwS2FeZ78mX9Ld+LpIWSqpKqvb29TSjJbG/jx49HUstv48ePb/Wu8L4YYZrxzVjVWRYDLN97YcRSYClApVKp28dssHbu3ElE6/+8pHr/NYaX98XI0owj+h5gcmF+ErB9gOVmZjaMmhH0K4CL80/fvAd4ISKeBlYCZ0nqyE/CnpUvMzOzYdRw6EbSHcAsYIKkHrJP0hwEEBFfAe4GzgG2AC8Dl+VtOyR9Blidr2pxRAx0UtfMhkksGgfdh7W6jKwOG3Jqh3G6okqlEr56pQ0FSW0zLt3qOtqhhnaqIwWS1kREpV6bvxlrZpY4B72ZWeLa7odHzGx4tMNHGzs6/GX54eCgNxuBmjEu7vH1/YeHbszMEucjejPbS9lhnUb9fMTfHhz0NmL4s+PlOaDT4qC3EUPXvdgWASaJ6G51FTaSeIzezCxxDnozs8Q56M3MEuegNzNLnIPezCxxDnozs8Q56M3MEuegNzNLnIPezCxxpYJe0mxJmyRtkXR1nfYpklZJWi/pfkmTCm2flbRB0kZJX1Q7XBvVzGwEaRj0kkYBS4A5QBewQFJXTbcbgVsjYjqwGLghv+97gd8EpgMnAKcAM5tWvZmZNVTmiH4GsCUitkbELmAZMLemTxewKp++r9AewCHAaOBgsh8V/9lgizYzs/LKBP1EYFthvidfVrQOmJdPnweMlXRERDxAFvxP57eVEbGx9gEkLZRUlVTt7e3d120wK01Sy2/+VSUbbmWCvt6Yeu0lAK8CZkpaSzY08xSwW9KvAccDk8heHM6QdPpeK4tYGhGViKh0dnbu0waYlRURg741Yz07duxo8Z6wkabMZYp7gMmF+UnA9mKHiNgOnA8gaQwwLyJekLQQeDAiXsrb/hF4D/D9JtRu1lT+sQ1LVZkj+tXANEnHShoNzAdWFDtImiCpb13XADfn00+SHekfKOkgsqP9vYZuzNpBM474HfLWjhoGfUTsBq4AVpKF9PKI2CBpsaRz826zgE2SNgNHAtfny78FPA48TDaOvy4ivtPcTTAzs4Go3Y5AKpVKVKvVVpdhZrZfkbQmIir12vzNWDOzxDnozcwS56A3M0ucg97MLHEOejOzxDnozcwS56A3M0ucg97MLHEOejOzxJW5qJmZ2YjVrB/Fa+VVCBz0ZmYDKBPQktr6gnYeujEzS5yD3swscQ56M7PEOejNzBLnoDczS5yD3sxGtPHjxyNpUDdg0OsYP378kG1jqaCXNFvSJklbJF1dp32KpFWS1ku6X9KkQtsxku6RtFHSI5KmNq98M7PB2blzZ9N+L3gwt507dw7ZNjYMekmjgCXAHKALWCCpq6bbjcCtETEdWAzcUGi7FfjziDgemAE824zCzcysnDJH9DOALRGxNSJ2AcuAuTV9uoBV+fR9fe35C8KBEXEvQES8FBEvN6VyMzMrpUzQTwS2FeZ78mVF64B5+fR5wFhJRwDHAT+XdKektZL+PH+HYGZmw6RM0Ne70EPtd32vAmZKWgvMBJ4CdpNdYuG0vP0U4G3ApXs9gLRQUlVStbe3t3z1ZmbWUJmg7wEmF+YnAduLHSJie0ScHxEnAX+UL3shv+/afNhnN/Bt4N21DxARSyOiEhGVzs7ON7gpZmZWT5mgXw1Mk3SspNHAfGBFsYOkCZL61nUNcHPhvh2S+tL7DOCRwZc9eIP9KFSzrmhnZjbUGgZ9fiR+BbAS2Agsj4gNkhZLOjfvNgvYJGkzcCRwfX7fV8mGbVZJephsGOirTd+KN6DRR53K9jEza3dqt8CqVCpRrVZbXUbbX3bUzJqjXf6vD7YOSWsiolKvLclvxo6Eb7qZmZWV5A+P9H3TrdU8jm9m7SDJoLd904wXpHZ4YTWz+hz01jCk22UM08zemCTH6M3MbA8HfeJ8YtrMPHSTOJ+YNjMf0ZuZJc5Bb2aWOAe9mVniHPRmZolz0JuZJc5Bb2aWOAe9mVniHPRmZolz0JuZJc5Bb2aWuCQvgRCLxkH3Ya0uI6vDzKzFSgW9pNnAXwCjgL+OiD+raZ9C9oPgncAO4KKI6Cm0jyP7vdm7IuKKJtXef73Xvdg213eJ7lZXYWYjXcOhG0mjgCXAHKALWCCpq6bbjcCtETEdWAzcUNP+GeB7gy/XzMz2VZkx+hnAlojYGhG7gGXA3Jo+XcCqfPq+Yrukk4EjgXsGX66Zme2rMkE/EdhWmO/JlxWtA+bl0+cBYyUdIekA4HPAJwd6AEkLJVUlVXt7e8tVbmZmpZQJ+noXEq8dAL8KmClpLTATeArYDXwUuDsitjGAiFgaEZWIqHR2dpYoyczMyipzMrYHmFyYnwRsL3aIiO3A+QCSxgDzIuIFSb8BnCbpo8AYYLSklyLi6qZUb2ZmDZUJ+tXANEnHkh2pzwc+VOwgaQKwIyJeA64h+wQOEXFhoc+lQMUhb2Y2vBoO3UTEbuAKYCXZRySXR8QGSYslnZt3mwVskrSZ7MTr9UNUr5mZ7SO1w+fNiyqVSlSr1UGtQ1L7fI6+xXW0Qw3tVIdZrXb52xxsHZLWRESlXpsvgWBmljgHvZlZ4hz0ZmaJc9CbmSXOQW9mljgHvZlZ4hz0ZmaJc9CbmSXOQW9mljgHvZlZ4hz0ZmaJS/LHwSG7bkSrdXR0tLoEM7M0g74ZFyhqlwsdmZkNloduzMwSl+QRvZlZWbFoHHQf1uoysjqGiIPezEY0XfdiWwzTSiK6h2bdIzboy5ysbdSnHf44zMwaKTVGL2m2pE2Stkja6zdfJU2RtErSekn3S5qULz9R0gOSNuRtFzR7A96oiBj0zcxsf9Aw6CWNApYAc4AuYIGkrppuNwK3RsR0YDFwQ778ZeDiiPh1YDZwk6TDm1W8mZk1VuaIfgawJSK2RsQuYBkwt6ZPF7Aqn76vrz0iNkfEY/n0duBZoLMZhZuZWTllgn4isK0w35MvK1oHzMunzwPGSjqi2EHSDGA08HjtA0haKKkqqdrb21u2djMzK6FM0Nc7I1k7QH0VMFPSWmAm8BSw+1crkN4K3AZcFhGv7bWyiKURUYmISmenD/jNzJqpzKdueoDJhflJwPZih3xY5nwASWOAeRHxQj4/DvgH4NqIeLAZRZuZWXlljuhXA9MkHStpNDAfWFHsIGmCpL51XQPcnC8fDdxFdqL2m80r28zMymoY9BGxG7gCWAlsBJZHxAZJiyWdm3ebBWyStBk4Erg+X/67wOnApZIeym8nNnsjzMysf2q3z4NXKpWoVqutLiMZ7XJxtnapw6xWu/xtDrYOSWsiolKvzRc1MzNLnIPezCxxDnozs8Q56M3MEuegNzNLnIPezCxxDnozs8Q56M3MEuegNzNLnIPezCxxDnozs8Q56M3MEuegNzNLnIPezCxxZX5hyvZjsWgcdB/W6jKyOsysJRz0idN1L7bPtba7W12F2cjkoRszs8SVCnpJsyVtkrRF0tV12qdIWiVpvaT7JU0qtF0i6bH8dkkzizczs8YaBr2kUcASYA7QBSyQ1FXT7UayHwCfDiwGbsjvOx5YBJwKzAAWSepoXvlmZtZImSP6GcCWiNgaEbuAZcDcmj5dwKp8+r5C+9nAvRGxIyJ2AvcCswdftpmZlVUm6CcC2wrzPfmyonXAvHz6PGCspCNK3hdJCyVVJVV7e3vL1m5mZiWUCXrVWVb7MY6rgJmS1gIzgaeA3SXvS0QsjYhKRFQ6OztLlGRmZmWV+XhlDzC5MD8J2F7sEBHbgfMBJI0B5kXEC5J6gFk1971/EPWamdk+KnNEvxqYJulYSaOB+cCKYgdJEyT1resa4OZ8eiVwlqSO/CTsWfkyMzMbJg2DPiJ2A1eQBfRGYHlEbJC0WNK5ebdZwCZJm4Ejgevz++4APkP2YrEaWJwvMzOzYaJ2+NZkUaVSiWq12uoykiGpfb4Z2wZ1mNVql7/NwdYhaU1EVOq1+ZuxZmaJc9CbmSXOQW9mljgHvZlZ4hz0ZmaJc9CbmSXOQW9mljgHvZlZ4hz0ZmaJc9CbmSXOQW9mljgHvZlZ4hz0ZmaJK/PDI2ZmSZPq/Rje8Oro6BiydTvozWxEa3Rp4Ga9CLTyUsgOejOzAbTDteoHy2P0ZmaJKxX0kmZL2iRpi6Sr67QfI+k+SWslrZd0Tr78IEn/V9LDkjZKuqbZG2BmZgNrGPSSRgFLgDlAF7BAUldNt2vJfkv2JLIfD/9SvvyDwMER8U7gZOAjkqY2p3QzMyujzBH9DGBLRGyNiF3AMmBuTZ8AxuXThwHbC8sPlXQg8CZgF/DioKs2M7PSygT9RGBbYb4nX1bUDVwkqQe4G/hYvvxbwC+Ap4EngRsjYsdgCjYzs31TJujrfbao9jT0AuCWiJgEnAPcJukAsncDrwJHA8cCn5D0tr0eQFooqSqp2tvbu08bYGZmAysT9D3A5ML8JPYMzfS5HFgOEBEPAIcAE4APAf8UEa9ExLPAj4BK7QNExNKIqEREpbOzc9+3wszM+lUm6FcD0yQdK2k02cnWFTV9ngTOBJB0PFnQ9+bLz1DmUOA9wKPNKt7MzBprGPQRsRu4AlgJbCT7dM0GSYslnZt3+wTwYUnrgDuASyP7lsESYAzwE7IXjK9FxPoh2A4zM+uH2u1bX5VKJarVaqvLSIaktvhmX7vUYZYqSWsiYq+hcfA3Y83MkuegNzNLnIPezCxxDnozs8Q56M3MEuegNzNLnIPezCxxDnozs8Q56M3MEuegNzNLnIPezCxxDnozs8Q56M3MEuegNzNLnIPezCxxDnozs8Q56M3MEuegNzNLXKmglzRb0iZJWyRdXaf9GEn3SVorab2kcwpt0yU9IGmDpIclHdLMDTAzs4Ed2KiDpFFkP/L920APsFrSioh4pNDtWrIfDf+ypC7gbmCqpAOB24Hfi4h1ko4AXmn6VpiZWb/KHNHPALZExNaI2AUsA+bW9AlgXD59GLA9nz4LWB8R6wAi4vmIeHXwZZuZWVllgn4isK0w35MvK+oGLpLUQ3Y0/7F8+XFASFop6ceSPlXvASQtlFSVVO3t7d2nDTAzs4GVCXrVWRY18wuAWyJiEnAOcJukA8iGhn4LuDD/9zxJZ+61soilEVGJiEpnZ+c+bYCZmQ2sTND3AJML85PYMzTT53JgOUBEPAAcAkzI7/u9iHguIl4mO9p/92CLNjOz8soE/WpgmqRjJY0G5gMravo8CZwJIOl4sqDvBVYC0yW9OT8xOxN4BBtWklp+6+joaPVuMBuxGn7qJiJ2S7qCLLRHATdHxAZJi4FqRKwAPgF8VdKVZMM6l0ZEADslfZ7sxSKAuyPiH4ZqY2xv2dMwOJKash4zaw2123/gSqUS1Wq11WVYgYPerP1JWhMRlXpt/masmVniHPRmZolz0JuZJc5Bb2aWOAe9mVniHPRmZolz0JuZJc5Bb2aWuIbfjLX0SfWuW7dvffyFKrP25aA3h7RZ4jx0Y2aWOAe9mVniHPRmZolz0JuZJc5Bb2aWOAe9mVniHPRmZolz0JuZJa7tfkpQUi/wRKvrACYAz7W6iDbhfbGH98Ue3hd7tMO+mBIRnfUa2i7o24Wkan+/vzjSeF/s4X2xh/fFHu2+Lzx0Y2aWOAe9mVniHPT9W9rqAtqI98Ue3hd7eF/s0db7wmP0ZmaJ8xG9mVniHPSApJfqLOuW9JSkhyQ9ImlBK2obaiW2/TFJd0rqqunTKekVSR8ZvmqHTnE/SDon3+5j8n3xsqS39NM3JH2uMH+VpO5hK7yJJB0laZmkx/O/+bslHZe3XSnpPyQdVug/S9ILktZKelTSjfnyy/K/nYck7ZL0cD79Z63atmYZ6Pmu+X/zqKQvS2qLjG2LItrYFyLiRGAu8FeSDmp1QcPoCxFxYkRMA74BfFdS8TO6HwQeBJJ6AZR0JvCXwOyIeDJf/BzwiX7u8p/A+ZImDEd9Q0XZT4jdBdwfEW+PiC7g08CReZcFwGrgvJq7/iAiTgJOAt4v6Tcj4mv5386JwHbgffn81cOzNUOq0fPdlxldwDuBmcNW2QAc9CVExGPAy0BHq2tphYj4BnAP8KHC4gVk4TdJ0sSWFNZkkk4Dvgr814h4vNB0M3CBpPF17rab7ETclcNQ4lB6H/BKRHylb0FEPBQRP5D0dmAMcC39vLBHxC+Bh4Ak/hYGUPb5Hg0cAuwc8opKcNCXIOndwGMR8Wyra2mhHwP/BUDSZOCoiPhXYDlwQSsLa5KDgb8HPhARj9a0vUQW9v+7n/suAS4sDmvsh04A1vTTtgC4A/gB8I7iMFYfSR3ANOD7Q1Zh+xjo+b5S0kPA08DmiHhoeEurz0E/sCslbQL+BehucS2tVvx18PlkAQ+wjDSGb14B/hm4vJ/2LwKXSBpX2xARLwK3Ar8/dOW11HxgWUS8BtxJNmzX5zRJ64FngP8XEc+0osDh1OD57hu6eQtwqKT5w1pcPxz0A/tCRLyD7Ij1VkmHtLqgFjoJ2JhPLwAulfRvwArgXZKmtaqwJnkN+F3gFEmfrm2MiJ8Dfwt8tJ/730T2InHokFU4tDYAJ9culDSd7Ej93vz5ns/rX9h/EBHTycaj/6ekE4eh1nYw4PMdEa8A/wScPpxF9cdBX0JE3AlUgUtaXUsrSJoHnAXcIekdwKERMTEipkbEVOAGsgDYr0XEy8D7yd6W1zuy/zzwEeDAOvfdQfYup793BO3uu8DBkj7ct0DSKcBfAN19z3VEHA1MlDSleOeI2Ez2d/CHw1l0qzR6vvOT2+8FHq/XPtwc9Jk3S+op3D5ep89i4OPt8nGpJupv26/s+3glcBFwRkT0kh3N3VWzjr8jjeGbvv/As4FrJc2taXuObNsP7ufunyO7iuF+J7JvTp4H/Hb+8coNZMOVs9j7+b6L+i/sXwFOl3TsEJbaTuo9331j9D8hOyD40rBXVYe/GWtmlrjUjk7NzKyGg97MLHEOejOzxDnozcwS56A3M0ucg97MLHEOejOzxDnozcwS9/8BY6XLTmzaHPsAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "fig = plt.figure()\n",
    "fig.suptitle('Comparison of models')\n",
    "ax = fig.add_subplot(111)\n",
    "plt.boxplot(results)\n",
    "ax.set_xticklabels(names)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It is evident from this picture that CART has the smallest variability and the highest accuracy. So, we will use CART as our model of choice."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "#split dataset into test and train\n",
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = DecisionTreeClassifier()\n",
    "fitted_model = model.fit(X_train, y_train)\n",
    "y_predict = model.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9963135593220339\n",
      "F-1 score: 0.9963274093461101\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import f1_score\n",
    "\n",
    "print(\"Accuracy:\",accuracy_score(y_test, y_predict))\n",
    "print(\"F-1 score:\", f1_score(y_test, y_predict))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Prediction accuracy and F-1 is very high, but is our model is not predicting just one class?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Counter({1: 11888, 0: 11712})\n"
     ]
    }
   ],
   "source": [
    "import collections\n",
    "\n",
    "print(collections.Counter(y_predict))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finally, let's create a confusion matrix. Confusion matrix  tells contrasts prediction of different classes and allows to better evaluate the performance."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Predicted Surface Failure</th>\n",
       "      <th>Predicted Underground failure</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>True Surface failure</td>\n",
       "      <td>11712</td>\n",
       "      <td>87</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>True Underground failure</td>\n",
       "      <td>0</td>\n",
       "      <td>11801</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          Predicted Surface Failure  \\\n",
       "True Surface failure                          11712   \n",
       "True Underground failure                          0   \n",
       "\n",
       "                          Predicted Underground failure  \n",
       "True Surface failure                                 87  \n",
       "True Underground failure                          11801  "
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "pd.DataFrame(\n",
    "    confusion_matrix(y_test, y_predict),\n",
    "    columns=['Predicted Surface Failure', 'Predicted Underground failure'],\n",
    "    index=['True Surface failure', 'True Underground failure']\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Save the model to use it later."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "pkl_filename = \"pickled_decision_tree.pkl\"\n",
    "with open(pkl_filename, 'wb') as file:\n",
    "    pickle.dump(model, file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Applying the model to the test data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finally, let's make our predictions using test data. For that, we need to apply the same transformations to the test data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id                            int64\n",
       "sensor1_measure               int64\n",
       "sensor2_measure             float64\n",
       "sensor3_measure             float64\n",
       "sensor4_measure             float64\n",
       "                             ...   \n",
       "sensor105_histogram_bin7    float64\n",
       "sensor105_histogram_bin8    float64\n",
       "sensor105_histogram_bin9    float64\n",
       "sensor106_measure           float64\n",
       "sensor107_measure           float64\n",
       "Length: 171, dtype: object"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test2 = df_test\n",
    "\n",
    "df_test2 = df_test2.apply(pd.to_numeric, errors='coerce')\n",
    "df_test2.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "sensor7 = ['sensor7_histogram_bin0',\n",
    " 'sensor7_histogram_bin1',\n",
    " 'sensor7_histogram_bin2',\n",
    " 'sensor7_histogram_bin3',\n",
    " 'sensor7_histogram_bin4',\n",
    " 'sensor7_histogram_bin5',\n",
    " 'sensor7_histogram_bin6',\n",
    " 'sensor7_histogram_bin7',\n",
    " 'sensor7_histogram_bin8',\n",
    " 'sensor7_histogram_bin9']\n",
    "sensor24 = ['sensor24_histogram_bin0',\n",
    " 'sensor24_histogram_bin1',\n",
    " 'sensor24_histogram_bin2',\n",
    " 'sensor24_histogram_bin3',\n",
    " 'sensor24_histogram_bin4',\n",
    " 'sensor24_histogram_bin5',\n",
    " 'sensor24_histogram_bin6',\n",
    " 'sensor24_histogram_bin7',\n",
    " 'sensor24_histogram_bin8',\n",
    " 'sensor24_histogram_bin9']\n",
    "sensor25 = ['sensor25_histogram_bin0',\n",
    " 'sensor25_histogram_bin1',\n",
    " 'sensor25_histogram_bin2',\n",
    " 'sensor25_histogram_bin3',\n",
    " 'sensor25_histogram_bin4',\n",
    " 'sensor25_histogram_bin5',\n",
    " 'sensor25_histogram_bin6',\n",
    " 'sensor25_histogram_bin7',\n",
    " 'sensor25_histogram_bin8',\n",
    " 'sensor25_histogram_bin9']\n",
    "sensor26 = ['sensor26_histogram_bin0',\n",
    " 'sensor26_histogram_bin1',\n",
    " 'sensor26_histogram_bin2',\n",
    " 'sensor26_histogram_bin3',\n",
    " 'sensor26_histogram_bin4',\n",
    " 'sensor26_histogram_bin5',\n",
    " 'sensor26_histogram_bin6',\n",
    " 'sensor26_histogram_bin7',\n",
    " 'sensor26_histogram_bin8',\n",
    " 'sensor26_histogram_bin9']\n",
    "sensor64 = ['sensor64_histogram_bin0',\n",
    " 'sensor64_histogram_bin1',\n",
    " 'sensor64_histogram_bin2',\n",
    " 'sensor64_histogram_bin3',\n",
    " 'sensor64_histogram_bin4',\n",
    " 'sensor64_histogram_bin5',\n",
    " 'sensor64_histogram_bin6',\n",
    " 'sensor64_histogram_bin7',\n",
    " 'sensor64_histogram_bin8',\n",
    " 'sensor64_histogram_bin9']\n",
    "sensor69 = ['sensor69_histogram_bin0',\n",
    " 'sensor69_histogram_bin1',\n",
    " 'sensor69_histogram_bin2',\n",
    " 'sensor69_histogram_bin3',\n",
    " 'sensor69_histogram_bin4',\n",
    " 'sensor69_histogram_bin5',\n",
    " 'sensor69_histogram_bin6',\n",
    " 'sensor69_histogram_bin7',\n",
    " 'sensor69_histogram_bin8',\n",
    " 'sensor69_histogram_bin9']\n",
    "sensor105 = ['sensor105_histogram_bin0',\n",
    " 'sensor105_histogram_bin1',\n",
    " 'sensor105_histogram_bin2',\n",
    " 'sensor105_histogram_bin3',\n",
    " 'sensor105_histogram_bin4',\n",
    " 'sensor105_histogram_bin5',\n",
    " 'sensor105_histogram_bin6',\n",
    " 'sensor105_histogram_bin7',\n",
    " 'sensor105_histogram_bin8',\n",
    " 'sensor105_histogram_bin9']\n",
    "df_test2['sensor7_average'] = df_test2[sensor7].mean(axis=1)\n",
    "df_test2['sensor24_average'] = df_test2[sensor24].mean(axis=1)\n",
    "df_test2['sensor25_average'] = df_test2[sensor25].mean(axis=1)\n",
    "df_test2['sensor26_average'] = df_test2[sensor26].mean(axis=1)\n",
    "df_test2['sensor64_average'] = df_test2[sensor64].mean(axis=1)\n",
    "df_test2['sensor69_average'] = df_test2[sensor69].mean(axis=1)\n",
    "df_test2['sensor105_average'] = df_test2[sensor105].mean(axis=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['id',\n",
       " 'sensor1_measure',\n",
       " 'sensor2_measure',\n",
       " 'sensor3_measure',\n",
       " 'sensor4_measure',\n",
       " 'sensor5_measure',\n",
       " 'sensor6_measure',\n",
       " 'sensor7_histogram_bin0',\n",
       " 'sensor7_histogram_bin1',\n",
       " 'sensor7_histogram_bin2',\n",
       " 'sensor7_histogram_bin3',\n",
       " 'sensor7_histogram_bin4',\n",
       " 'sensor7_histogram_bin5',\n",
       " 'sensor7_histogram_bin6',\n",
       " 'sensor7_histogram_bin7',\n",
       " 'sensor7_histogram_bin8',\n",
       " 'sensor7_histogram_bin9',\n",
       " 'sensor8_measure',\n",
       " 'sensor9_measure',\n",
       " 'sensor10_measure',\n",
       " 'sensor11_measure',\n",
       " 'sensor12_measure',\n",
       " 'sensor13_measure',\n",
       " 'sensor14_measure',\n",
       " 'sensor15_measure',\n",
       " 'sensor16_measure',\n",
       " 'sensor17_measure',\n",
       " 'sensor18_measure',\n",
       " 'sensor19_measure',\n",
       " 'sensor20_measure',\n",
       " 'sensor21_measure',\n",
       " 'sensor22_measure',\n",
       " 'sensor23_measure',\n",
       " 'sensor24_histogram_bin0',\n",
       " 'sensor24_histogram_bin1',\n",
       " 'sensor24_histogram_bin2',\n",
       " 'sensor24_histogram_bin3',\n",
       " 'sensor24_histogram_bin4',\n",
       " 'sensor24_histogram_bin5',\n",
       " 'sensor24_histogram_bin6',\n",
       " 'sensor24_histogram_bin7',\n",
       " 'sensor24_histogram_bin8',\n",
       " 'sensor24_histogram_bin9',\n",
       " 'sensor25_histogram_bin0',\n",
       " 'sensor25_histogram_bin1',\n",
       " 'sensor25_histogram_bin2',\n",
       " 'sensor25_histogram_bin3',\n",
       " 'sensor25_histogram_bin4',\n",
       " 'sensor25_histogram_bin5',\n",
       " 'sensor25_histogram_bin6',\n",
       " 'sensor25_histogram_bin7',\n",
       " 'sensor25_histogram_bin8',\n",
       " 'sensor25_histogram_bin9',\n",
       " 'sensor26_histogram_bin0',\n",
       " 'sensor26_histogram_bin1',\n",
       " 'sensor26_histogram_bin2',\n",
       " 'sensor26_histogram_bin3',\n",
       " 'sensor26_histogram_bin4',\n",
       " 'sensor26_histogram_bin5',\n",
       " 'sensor26_histogram_bin6',\n",
       " 'sensor26_histogram_bin7',\n",
       " 'sensor26_histogram_bin8',\n",
       " 'sensor26_histogram_bin9',\n",
       " 'sensor27_measure',\n",
       " 'sensor28_measure',\n",
       " 'sensor29_measure',\n",
       " 'sensor30_measure',\n",
       " 'sensor31_measure',\n",
       " 'sensor32_measure',\n",
       " 'sensor33_measure',\n",
       " 'sensor34_measure',\n",
       " 'sensor35_measure',\n",
       " 'sensor36_measure',\n",
       " 'sensor37_measure',\n",
       " 'sensor38_measure',\n",
       " 'sensor39_measure',\n",
       " 'sensor40_measure',\n",
       " 'sensor41_measure',\n",
       " 'sensor42_measure',\n",
       " 'sensor43_measure',\n",
       " 'sensor44_measure',\n",
       " 'sensor45_measure',\n",
       " 'sensor46_measure',\n",
       " 'sensor47_measure',\n",
       " 'sensor48_measure',\n",
       " 'sensor49_measure',\n",
       " 'sensor50_measure',\n",
       " 'sensor51_measure',\n",
       " 'sensor52_measure',\n",
       " 'sensor53_measure',\n",
       " 'sensor54_measure',\n",
       " 'sensor55_measure',\n",
       " 'sensor56_measure',\n",
       " 'sensor57_measure',\n",
       " 'sensor58_measure',\n",
       " 'sensor59_measure',\n",
       " 'sensor60_measure',\n",
       " 'sensor61_measure',\n",
       " 'sensor62_measure',\n",
       " 'sensor63_measure',\n",
       " 'sensor64_histogram_bin0',\n",
       " 'sensor64_histogram_bin1',\n",
       " 'sensor64_histogram_bin2',\n",
       " 'sensor64_histogram_bin3',\n",
       " 'sensor64_histogram_bin4',\n",
       " 'sensor64_histogram_bin5',\n",
       " 'sensor64_histogram_bin6',\n",
       " 'sensor64_histogram_bin7',\n",
       " 'sensor64_histogram_bin8',\n",
       " 'sensor64_histogram_bin9',\n",
       " 'sensor65_measure',\n",
       " 'sensor66_measure',\n",
       " 'sensor67_measure',\n",
       " 'sensor68_measure',\n",
       " 'sensor69_histogram_bin0',\n",
       " 'sensor69_histogram_bin1',\n",
       " 'sensor69_histogram_bin2',\n",
       " 'sensor69_histogram_bin3',\n",
       " 'sensor69_histogram_bin4',\n",
       " 'sensor69_histogram_bin5',\n",
       " 'sensor69_histogram_bin6',\n",
       " 'sensor69_histogram_bin7',\n",
       " 'sensor69_histogram_bin8',\n",
       " 'sensor69_histogram_bin9',\n",
       " 'sensor70_measure',\n",
       " 'sensor71_measure',\n",
       " 'sensor72_measure',\n",
       " 'sensor73_measure',\n",
       " 'sensor74_measure',\n",
       " 'sensor75_measure',\n",
       " 'sensor76_measure',\n",
       " 'sensor77_measure',\n",
       " 'sensor78_measure',\n",
       " 'sensor79_measure',\n",
       " 'sensor80_measure',\n",
       " 'sensor81_measure',\n",
       " 'sensor82_measure',\n",
       " 'sensor83_measure',\n",
       " 'sensor84_measure',\n",
       " 'sensor85_measure',\n",
       " 'sensor86_measure',\n",
       " 'sensor87_measure',\n",
       " 'sensor88_measure',\n",
       " 'sensor89_measure',\n",
       " 'sensor90_measure',\n",
       " 'sensor91_measure',\n",
       " 'sensor92_measure',\n",
       " 'sensor93_measure',\n",
       " 'sensor94_measure',\n",
       " 'sensor95_measure',\n",
       " 'sensor96_measure',\n",
       " 'sensor97_measure',\n",
       " 'sensor98_measure',\n",
       " 'sensor99_measure',\n",
       " 'sensor100_measure',\n",
       " 'sensor101_measure',\n",
       " 'sensor102_measure',\n",
       " 'sensor103_measure',\n",
       " 'sensor104_measure',\n",
       " 'sensor105_histogram_bin0',\n",
       " 'sensor105_histogram_bin1',\n",
       " 'sensor105_histogram_bin2',\n",
       " 'sensor105_histogram_bin3',\n",
       " 'sensor105_histogram_bin4',\n",
       " 'sensor105_histogram_bin5',\n",
       " 'sensor105_histogram_bin6',\n",
       " 'sensor105_histogram_bin7',\n",
       " 'sensor105_histogram_bin8',\n",
       " 'sensor105_histogram_bin9',\n",
       " 'sensor106_measure',\n",
       " 'sensor107_measure',\n",
       " 'sensor7_average',\n",
       " 'sensor24_average',\n",
       " 'sensor25_average',\n",
       " 'sensor26_average',\n",
       " 'sensor64_average',\n",
       " 'sensor69_average',\n",
       " 'sensor105_average']"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(df_test2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>sensor1_measure</th>\n",
       "      <th>sensor2_measure</th>\n",
       "      <th>sensor3_measure</th>\n",
       "      <th>sensor4_measure</th>\n",
       "      <th>sensor5_measure</th>\n",
       "      <th>sensor6_measure</th>\n",
       "      <th>sensor8_measure</th>\n",
       "      <th>sensor9_measure</th>\n",
       "      <th>sensor10_measure</th>\n",
       "      <th>...</th>\n",
       "      <th>sensor104_measure</th>\n",
       "      <th>sensor106_measure</th>\n",
       "      <th>sensor107_measure</th>\n",
       "      <th>sensor7_average</th>\n",
       "      <th>sensor24_average</th>\n",
       "      <th>sensor25_average</th>\n",
       "      <th>sensor26_average</th>\n",
       "      <th>sensor64_average</th>\n",
       "      <th>sensor69_average</th>\n",
       "      <th>sensor105_average</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>66888</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.130706e+09</td>\n",
       "      <td>332.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1967396.0</td>\n",
       "      <td>119208.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1134.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>446803.0</td>\n",
       "      <td>446803.0</td>\n",
       "      <td>446803.0</td>\n",
       "      <td>446803.0</td>\n",
       "      <td>446803.0</td>\n",
       "      <td>446803.0</td>\n",
       "      <td>446803.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>91122</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3068120.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>623296.4</td>\n",
       "      <td>623296.4</td>\n",
       "      <td>623296.4</td>\n",
       "      <td>623296.4</td>\n",
       "      <td>623296.4</td>\n",
       "      <td>623296.4</td>\n",
       "      <td>623296.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>218924</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>224.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1227540.6</td>\n",
       "      <td>1227540.6</td>\n",
       "      <td>1227540.6</td>\n",
       "      <td>1227540.6</td>\n",
       "      <td>1227540.6</td>\n",
       "      <td>1227540.6</td>\n",
       "      <td>1227540.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>16</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.000000e+01</td>\n",
       "      <td>28.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2246.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>46.0</td>\n",
       "      <td>...</td>\n",
       "      <td>56.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>692.2</td>\n",
       "      <td>692.2</td>\n",
       "      <td>692.2</td>\n",
       "      <td>692.2</td>\n",
       "      <td>692.2</td>\n",
       "      <td>692.2</td>\n",
       "      <td>692.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>39084</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.054000e+03</td>\n",
       "      <td>1032.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1213570.0</td>\n",
       "      <td>15120.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>2120.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>239978.2</td>\n",
       "      <td>239978.2</td>\n",
       "      <td>239978.2</td>\n",
       "      <td>239978.2</td>\n",
       "      <td>239978.2</td>\n",
       "      <td>239978.2</td>\n",
       "      <td>239978.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15996</td>\n",
       "      <td>15997</td>\n",
       "      <td>40084</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.200000e+02</td>\n",
       "      <td>162.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1289494.0</td>\n",
       "      <td>7152.0</td>\n",
       "      <td>56.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1192.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>256450.4</td>\n",
       "      <td>256450.4</td>\n",
       "      <td>256450.4</td>\n",
       "      <td>256450.4</td>\n",
       "      <td>256450.4</td>\n",
       "      <td>256450.4</td>\n",
       "      <td>256450.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15997</td>\n",
       "      <td>15998</td>\n",
       "      <td>102790</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.780000e+02</td>\n",
       "      <td>502.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2728194.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>38.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>563989.4</td>\n",
       "      <td>563989.4</td>\n",
       "      <td>563989.4</td>\n",
       "      <td>563989.4</td>\n",
       "      <td>563989.4</td>\n",
       "      <td>563989.4</td>\n",
       "      <td>563989.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15998</td>\n",
       "      <td>15999</td>\n",
       "      <td>50846</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.700000e+02</td>\n",
       "      <td>758.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1925202.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1758.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>377912.8</td>\n",
       "      <td>377912.8</td>\n",
       "      <td>377912.8</td>\n",
       "      <td>377912.8</td>\n",
       "      <td>377912.8</td>\n",
       "      <td>377912.8</td>\n",
       "      <td>377912.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15999</td>\n",
       "      <td>16000</td>\n",
       "      <td>48</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.130706e+09</td>\n",
       "      <td>70.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5534.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>58.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1523.8</td>\n",
       "      <td>1523.8</td>\n",
       "      <td>1523.8</td>\n",
       "      <td>1523.8</td>\n",
       "      <td>1523.8</td>\n",
       "      <td>1523.8</td>\n",
       "      <td>1523.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>16000</td>\n",
       "      <td>16001</td>\n",
       "      <td>60000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.130706e+09</td>\n",
       "      <td>558.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1479170.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>156.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1600.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>325899.8</td>\n",
       "      <td>325899.8</td>\n",
       "      <td>325899.8</td>\n",
       "      <td>325899.8</td>\n",
       "      <td>325899.8</td>\n",
       "      <td>325899.8</td>\n",
       "      <td>325899.8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>16001 rows × 108 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          id  sensor1_measure  sensor2_measure  sensor3_measure  \\\n",
       "0          1            66888              0.0     2.130706e+09   \n",
       "1          2            91122              0.0     0.000000e+00   \n",
       "2          3           218924              0.0     0.000000e+00   \n",
       "3          4               16              0.0     3.000000e+01   \n",
       "4          5            39084              0.0     1.054000e+03   \n",
       "...      ...              ...              ...              ...   \n",
       "15996  15997            40084              0.0     2.200000e+02   \n",
       "15997  15998           102790              0.0     8.780000e+02   \n",
       "15998  15999            50846              0.0     8.700000e+02   \n",
       "15999  16000               48              0.0     2.130706e+09   \n",
       "16000  16001            60000              0.0     2.130706e+09   \n",
       "\n",
       "       sensor4_measure  sensor5_measure  sensor6_measure  sensor8_measure  \\\n",
       "0                332.0              0.0              0.0        1967396.0   \n",
       "1                  0.0              0.0              0.0        3068120.0   \n",
       "2                  0.0              0.0              0.0              0.0   \n",
       "3                 28.0              0.0              0.0           2246.0   \n",
       "4               1032.0              0.0              0.0        1213570.0   \n",
       "...                ...              ...              ...              ...   \n",
       "15996            162.0              0.0              0.0        1289494.0   \n",
       "15997            502.0              0.0              0.0        2728194.0   \n",
       "15998            758.0              0.0              0.0        1925202.0   \n",
       "15999             70.0              0.0              0.0           5534.0   \n",
       "16000            558.0              0.0              0.0        1479170.0   \n",
       "\n",
       "       sensor9_measure  sensor10_measure  ...  sensor104_measure  \\\n",
       "0             119208.0               0.0  ...             1134.0   \n",
       "1                  0.0               0.0  ...                0.0   \n",
       "2                  0.0             224.0  ...                0.0   \n",
       "3                  0.0              46.0  ...               56.0   \n",
       "4              15120.0               0.0  ...             2120.0   \n",
       "...                ...               ...  ...                ...   \n",
       "15996           7152.0              56.0  ...             1192.0   \n",
       "15997              0.0              38.0  ...                0.0   \n",
       "15998              0.0               0.0  ...             1758.0   \n",
       "15999              0.0               0.0  ...               58.0   \n",
       "16000              0.0             156.0  ...             1600.0   \n",
       "\n",
       "       sensor106_measure  sensor107_measure  sensor7_average  \\\n",
       "0                    0.0                0.0         446803.0   \n",
       "1                    0.0                0.0         623296.4   \n",
       "2                    0.0                0.0        1227540.6   \n",
       "3                    0.0                0.0            692.2   \n",
       "4                    0.0                0.0         239978.2   \n",
       "...                  ...                ...              ...   \n",
       "15996                0.0                0.0         256450.4   \n",
       "15997                0.0                0.0         563989.4   \n",
       "15998                0.0                0.0         377912.8   \n",
       "15999                0.0                0.0           1523.8   \n",
       "16000                0.0                0.0         325899.8   \n",
       "\n",
       "       sensor24_average  sensor25_average  sensor26_average  sensor64_average  \\\n",
       "0              446803.0          446803.0          446803.0          446803.0   \n",
       "1              623296.4          623296.4          623296.4          623296.4   \n",
       "2             1227540.6         1227540.6         1227540.6         1227540.6   \n",
       "3                 692.2             692.2             692.2             692.2   \n",
       "4              239978.2          239978.2          239978.2          239978.2   \n",
       "...                 ...               ...               ...               ...   \n",
       "15996          256450.4          256450.4          256450.4          256450.4   \n",
       "15997          563989.4          563989.4          563989.4          563989.4   \n",
       "15998          377912.8          377912.8          377912.8          377912.8   \n",
       "15999            1523.8            1523.8            1523.8            1523.8   \n",
       "16000          325899.8          325899.8          325899.8          325899.8   \n",
       "\n",
       "       sensor69_average  sensor105_average  \n",
       "0              446803.0           446803.0  \n",
       "1              623296.4           623296.4  \n",
       "2             1227540.6          1227540.6  \n",
       "3                 692.2              692.2  \n",
       "4              239978.2           239978.2  \n",
       "...                 ...                ...  \n",
       "15996          256450.4           256450.4  \n",
       "15997          563989.4           563989.4  \n",
       "15998          377912.8           377912.8  \n",
       "15999            1523.8             1523.8  \n",
       "16000          325899.8           325899.8  \n",
       "\n",
       "[16001 rows x 108 columns]"
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test3 = df_test2.fillna(0)\n",
    "not_hists2 = [col for col in df_test3.columns if not  'histogram' in col]\n",
    "df_test4 = df_test3[not_hists2]\n",
    "df_test4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test5 = df_test4.drop(['id'], axis=1)\n",
    "predictions = model.predict(df_test5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's check how many instances of each class were predicted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({0: 15695, 1: 306})"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import collections\n",
    "\n",
    "collections.Counter(predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "16001"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#sanity check:\n",
    "15695 + 306"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, ..., 0, 0, 0])"
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Save the output in submission-ready format:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [],
   "source": [
    "res_series = pd.Series(predictions)\n",
    "headers = ['target']\n",
    "res_series.index += 1 \n",
    "res_series.index.name = 'id'\n",
    "res_series.to_csv('submission5a.csv', header = headers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}