hyperparameter-tuning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "d82c1d9336f335adc3f0dececefdb0f97c8c745c"
   },
   "source": [
    "# Generalized Linear Models and Hyperparameter Tuning\n",
    "\n",
    "In this notebook we'll explore modules in sklearn's linear_model module and attempt to optimize the top performing models with hyperparameter tuning.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "from sklearn.model_selection import train_test_split, cross_val_score\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "from sklearn import linear_model\n",
    "from sklearn.model_selection import GridSearchCV, RandomizedSearchCV\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "np.random.seed(27)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [],
   "source": [
    "# setting up default plotting parameters\n",
    "%matplotlib inline\n",
    "\n",
    "plt.rcParams['figure.figsize'] = [20.0, 7.0]\n",
    "plt.rcParams.update({'font.size': 22,})\n",
    "\n",
    "sns.set_palette('viridis')\n",
    "sns.set_style('white')\n",
    "sns.set_context('talk', font_scale=0.8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "_uuid": "f3368d6319e979c81c215f52454798c6bcec5810"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Shape:  (250, 302)\n",
      "Test Shape:  (19750, 301)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>target</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "      <th>18</th>\n",
       "      <th>19</th>\n",
       "      <th>20</th>\n",
       "      <th>21</th>\n",
       "      <th>22</th>\n",
       "      <th>23</th>\n",
       "      <th>24</th>\n",
       "      <th>25</th>\n",
       "      <th>26</th>\n",
       "      <th>27</th>\n",
       "      <th>28</th>\n",
       "      <th>29</th>\n",
       "      <th>30</th>\n",
       "      <th>31</th>\n",
       "      <th>32</th>\n",
       "      <th>33</th>\n",
       "      <th>34</th>\n",
       "      <th>35</th>\n",
       "      <th>36</th>\n",
       "      <th>37</th>\n",
       "      <th>...</th>\n",
       "      <th>260</th>\n",
       "      <th>261</th>\n",
       "      <th>262</th>\n",
       "      <th>263</th>\n",
       "      <th>264</th>\n",
       "      <th>265</th>\n",
       "      <th>266</th>\n",
       "      <th>267</th>\n",
       "      <th>268</th>\n",
       "      <th>269</th>\n",
       "      <th>270</th>\n",
       "      <th>271</th>\n",
       "      <th>272</th>\n",
       "      <th>273</th>\n",
       "      <th>274</th>\n",
       "      <th>275</th>\n",
       "      <th>276</th>\n",
       "      <th>277</th>\n",
       "      <th>278</th>\n",
       "      <th>279</th>\n",
       "      <th>280</th>\n",
       "      <th>281</th>\n",
       "      <th>282</th>\n",
       "      <th>283</th>\n",
       "      <th>284</th>\n",
       "      <th>285</th>\n",
       "      <th>286</th>\n",
       "      <th>287</th>\n",
       "      <th>288</th>\n",
       "      <th>289</th>\n",
       "      <th>290</th>\n",
       "      <th>291</th>\n",
       "      <th>292</th>\n",
       "      <th>293</th>\n",
       "      <th>294</th>\n",
       "      <th>295</th>\n",
       "      <th>296</th>\n",
       "      <th>297</th>\n",
       "      <th>298</th>\n",
       "      <th>299</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-0.098</td>\n",
       "      <td>2.165</td>\n",
       "      <td>0.681</td>\n",
       "      <td>-0.614</td>\n",
       "      <td>1.309</td>\n",
       "      <td>-0.455</td>\n",
       "      <td>-0.236</td>\n",
       "      <td>0.276</td>\n",
       "      <td>-2.246</td>\n",
       "      <td>1.825</td>\n",
       "      <td>-0.912</td>\n",
       "      <td>-0.107</td>\n",
       "      <td>0.305</td>\n",
       "      <td>0.102</td>\n",
       "      <td>0.826</td>\n",
       "      <td>0.417</td>\n",
       "      <td>0.177</td>\n",
       "      <td>-0.673</td>\n",
       "      <td>-0.503</td>\n",
       "      <td>1.864</td>\n",
       "      <td>0.410</td>\n",
       "      <td>-1.927</td>\n",
       "      <td>0.102</td>\n",
       "      <td>-0.931</td>\n",
       "      <td>1.763</td>\n",
       "      <td>1.449</td>\n",
       "      <td>-1.097</td>\n",
       "      <td>-0.686</td>\n",
       "      <td>-0.250</td>\n",
       "      <td>-1.859</td>\n",
       "      <td>1.125</td>\n",
       "      <td>1.009</td>\n",
       "      <td>-2.296</td>\n",
       "      <td>0.385</td>\n",
       "      <td>-0.876</td>\n",
       "      <td>1.528</td>\n",
       "      <td>-0.144</td>\n",
       "      <td>-1.078</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.681</td>\n",
       "      <td>1.250</td>\n",
       "      <td>-0.565</td>\n",
       "      <td>-1.318</td>\n",
       "      <td>-0.923</td>\n",
       "      <td>0.075</td>\n",
       "      <td>-0.704</td>\n",
       "      <td>2.457</td>\n",
       "      <td>0.771</td>\n",
       "      <td>-0.460</td>\n",
       "      <td>0.569</td>\n",
       "      <td>-1.320</td>\n",
       "      <td>-1.516</td>\n",
       "      <td>-2.145</td>\n",
       "      <td>-1.120</td>\n",
       "      <td>0.156</td>\n",
       "      <td>0.820</td>\n",
       "      <td>-1.049</td>\n",
       "      <td>-1.125</td>\n",
       "      <td>0.484</td>\n",
       "      <td>0.617</td>\n",
       "      <td>1.253</td>\n",
       "      <td>1.248</td>\n",
       "      <td>0.504</td>\n",
       "      <td>-0.802</td>\n",
       "      <td>-0.896</td>\n",
       "      <td>-1.793</td>\n",
       "      <td>-0.284</td>\n",
       "      <td>-0.601</td>\n",
       "      <td>0.569</td>\n",
       "      <td>0.867</td>\n",
       "      <td>1.347</td>\n",
       "      <td>0.504</td>\n",
       "      <td>-0.649</td>\n",
       "      <td>0.672</td>\n",
       "      <td>-2.097</td>\n",
       "      <td>1.051</td>\n",
       "      <td>-0.414</td>\n",
       "      <td>1.038</td>\n",
       "      <td>-1.065</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.081</td>\n",
       "      <td>-0.973</td>\n",
       "      <td>-0.383</td>\n",
       "      <td>0.326</td>\n",
       "      <td>-0.428</td>\n",
       "      <td>0.317</td>\n",
       "      <td>1.172</td>\n",
       "      <td>0.352</td>\n",
       "      <td>0.004</td>\n",
       "      <td>-0.291</td>\n",
       "      <td>2.907</td>\n",
       "      <td>1.085</td>\n",
       "      <td>2.144</td>\n",
       "      <td>1.540</td>\n",
       "      <td>0.584</td>\n",
       "      <td>1.133</td>\n",
       "      <td>1.098</td>\n",
       "      <td>-0.237</td>\n",
       "      <td>-0.498</td>\n",
       "      <td>0.283</td>\n",
       "      <td>-1.100</td>\n",
       "      <td>-0.417</td>\n",
       "      <td>1.382</td>\n",
       "      <td>-0.515</td>\n",
       "      <td>-1.519</td>\n",
       "      <td>0.619</td>\n",
       "      <td>-0.128</td>\n",
       "      <td>0.866</td>\n",
       "      <td>-0.540</td>\n",
       "      <td>1.238</td>\n",
       "      <td>-0.227</td>\n",
       "      <td>0.269</td>\n",
       "      <td>-0.390</td>\n",
       "      <td>-2.721</td>\n",
       "      <td>1.659</td>\n",
       "      <td>0.106</td>\n",
       "      <td>-0.121</td>\n",
       "      <td>1.719</td>\n",
       "      <td>...</td>\n",
       "      <td>0.971</td>\n",
       "      <td>-1.489</td>\n",
       "      <td>0.530</td>\n",
       "      <td>0.917</td>\n",
       "      <td>-0.094</td>\n",
       "      <td>-1.407</td>\n",
       "      <td>0.887</td>\n",
       "      <td>-0.104</td>\n",
       "      <td>-0.583</td>\n",
       "      <td>1.267</td>\n",
       "      <td>-1.667</td>\n",
       "      <td>-2.771</td>\n",
       "      <td>-0.516</td>\n",
       "      <td>1.312</td>\n",
       "      <td>0.491</td>\n",
       "      <td>0.932</td>\n",
       "      <td>2.064</td>\n",
       "      <td>0.422</td>\n",
       "      <td>1.215</td>\n",
       "      <td>2.012</td>\n",
       "      <td>0.043</td>\n",
       "      <td>-0.307</td>\n",
       "      <td>-0.059</td>\n",
       "      <td>1.121</td>\n",
       "      <td>1.333</td>\n",
       "      <td>0.211</td>\n",
       "      <td>1.753</td>\n",
       "      <td>0.053</td>\n",
       "      <td>1.274</td>\n",
       "      <td>-0.612</td>\n",
       "      <td>-0.165</td>\n",
       "      <td>-1.695</td>\n",
       "      <td>-1.257</td>\n",
       "      <td>1.359</td>\n",
       "      <td>-0.808</td>\n",
       "      <td>-1.624</td>\n",
       "      <td>-0.458</td>\n",
       "      <td>-1.099</td>\n",
       "      <td>-0.936</td>\n",
       "      <td>0.973</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-0.523</td>\n",
       "      <td>-0.089</td>\n",
       "      <td>-0.348</td>\n",
       "      <td>0.148</td>\n",
       "      <td>-0.022</td>\n",
       "      <td>0.404</td>\n",
       "      <td>-0.023</td>\n",
       "      <td>-0.172</td>\n",
       "      <td>0.137</td>\n",
       "      <td>0.183</td>\n",
       "      <td>0.459</td>\n",
       "      <td>0.478</td>\n",
       "      <td>-0.425</td>\n",
       "      <td>0.352</td>\n",
       "      <td>1.095</td>\n",
       "      <td>0.300</td>\n",
       "      <td>-1.044</td>\n",
       "      <td>0.270</td>\n",
       "      <td>-1.038</td>\n",
       "      <td>0.144</td>\n",
       "      <td>-1.658</td>\n",
       "      <td>-0.946</td>\n",
       "      <td>0.633</td>\n",
       "      <td>-0.772</td>\n",
       "      <td>1.786</td>\n",
       "      <td>0.136</td>\n",
       "      <td>-0.103</td>\n",
       "      <td>-1.223</td>\n",
       "      <td>2.273</td>\n",
       "      <td>0.055</td>\n",
       "      <td>-2.032</td>\n",
       "      <td>-0.452</td>\n",
       "      <td>0.064</td>\n",
       "      <td>0.924</td>\n",
       "      <td>-0.692</td>\n",
       "      <td>-0.067</td>\n",
       "      <td>-0.917</td>\n",
       "      <td>1.896</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.540</td>\n",
       "      <td>-0.299</td>\n",
       "      <td>1.074</td>\n",
       "      <td>-0.748</td>\n",
       "      <td>1.086</td>\n",
       "      <td>-0.766</td>\n",
       "      <td>-0.931</td>\n",
       "      <td>0.432</td>\n",
       "      <td>1.345</td>\n",
       "      <td>-0.491</td>\n",
       "      <td>-1.602</td>\n",
       "      <td>-0.727</td>\n",
       "      <td>0.346</td>\n",
       "      <td>0.780</td>\n",
       "      <td>-0.527</td>\n",
       "      <td>-1.122</td>\n",
       "      <td>-0.208</td>\n",
       "      <td>-0.730</td>\n",
       "      <td>-0.302</td>\n",
       "      <td>2.535</td>\n",
       "      <td>-1.045</td>\n",
       "      <td>0.037</td>\n",
       "      <td>0.020</td>\n",
       "      <td>1.373</td>\n",
       "      <td>0.456</td>\n",
       "      <td>-0.277</td>\n",
       "      <td>1.381</td>\n",
       "      <td>1.843</td>\n",
       "      <td>0.749</td>\n",
       "      <td>0.202</td>\n",
       "      <td>0.013</td>\n",
       "      <td>0.263</td>\n",
       "      <td>-1.222</td>\n",
       "      <td>0.726</td>\n",
       "      <td>1.444</td>\n",
       "      <td>-1.165</td>\n",
       "      <td>-1.544</td>\n",
       "      <td>0.004</td>\n",
       "      <td>0.800</td>\n",
       "      <td>-1.211</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.067</td>\n",
       "      <td>-0.021</td>\n",
       "      <td>0.392</td>\n",
       "      <td>-1.637</td>\n",
       "      <td>-0.446</td>\n",
       "      <td>-0.725</td>\n",
       "      <td>-1.035</td>\n",
       "      <td>0.834</td>\n",
       "      <td>0.503</td>\n",
       "      <td>0.274</td>\n",
       "      <td>0.335</td>\n",
       "      <td>-1.148</td>\n",
       "      <td>0.067</td>\n",
       "      <td>-1.010</td>\n",
       "      <td>1.048</td>\n",
       "      <td>-1.442</td>\n",
       "      <td>0.210</td>\n",
       "      <td>0.836</td>\n",
       "      <td>-0.326</td>\n",
       "      <td>0.716</td>\n",
       "      <td>-0.764</td>\n",
       "      <td>0.248</td>\n",
       "      <td>-1.308</td>\n",
       "      <td>2.127</td>\n",
       "      <td>0.365</td>\n",
       "      <td>0.296</td>\n",
       "      <td>-0.808</td>\n",
       "      <td>1.854</td>\n",
       "      <td>0.118</td>\n",
       "      <td>0.380</td>\n",
       "      <td>0.999</td>\n",
       "      <td>-1.171</td>\n",
       "      <td>2.798</td>\n",
       "      <td>0.394</td>\n",
       "      <td>-1.048</td>\n",
       "      <td>1.078</td>\n",
       "      <td>0.401</td>\n",
       "      <td>-0.486</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.083</td>\n",
       "      <td>-0.831</td>\n",
       "      <td>1.251</td>\n",
       "      <td>-0.206</td>\n",
       "      <td>-0.933</td>\n",
       "      <td>-1.215</td>\n",
       "      <td>0.281</td>\n",
       "      <td>0.512</td>\n",
       "      <td>-0.424</td>\n",
       "      <td>0.769</td>\n",
       "      <td>0.223</td>\n",
       "      <td>-0.710</td>\n",
       "      <td>2.725</td>\n",
       "      <td>0.176</td>\n",
       "      <td>0.845</td>\n",
       "      <td>-1.226</td>\n",
       "      <td>1.527</td>\n",
       "      <td>-1.701</td>\n",
       "      <td>0.597</td>\n",
       "      <td>0.150</td>\n",
       "      <td>1.864</td>\n",
       "      <td>0.322</td>\n",
       "      <td>-0.214</td>\n",
       "      <td>1.282</td>\n",
       "      <td>0.408</td>\n",
       "      <td>-0.910</td>\n",
       "      <td>1.020</td>\n",
       "      <td>-0.299</td>\n",
       "      <td>-1.574</td>\n",
       "      <td>-1.618</td>\n",
       "      <td>-0.404</td>\n",
       "      <td>0.640</td>\n",
       "      <td>-0.595</td>\n",
       "      <td>-0.966</td>\n",
       "      <td>0.900</td>\n",
       "      <td>0.467</td>\n",
       "      <td>-0.562</td>\n",
       "      <td>-0.254</td>\n",
       "      <td>-0.533</td>\n",
       "      <td>0.238</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.347</td>\n",
       "      <td>-0.831</td>\n",
       "      <td>0.511</td>\n",
       "      <td>-0.021</td>\n",
       "      <td>1.225</td>\n",
       "      <td>1.594</td>\n",
       "      <td>0.585</td>\n",
       "      <td>1.509</td>\n",
       "      <td>-0.012</td>\n",
       "      <td>2.198</td>\n",
       "      <td>0.190</td>\n",
       "      <td>0.453</td>\n",
       "      <td>0.494</td>\n",
       "      <td>1.478</td>\n",
       "      <td>-1.412</td>\n",
       "      <td>0.270</td>\n",
       "      <td>-1.312</td>\n",
       "      <td>-0.322</td>\n",
       "      <td>-0.688</td>\n",
       "      <td>-0.198</td>\n",
       "      <td>-0.285</td>\n",
       "      <td>1.042</td>\n",
       "      <td>-0.315</td>\n",
       "      <td>-0.478</td>\n",
       "      <td>0.024</td>\n",
       "      <td>-0.190</td>\n",
       "      <td>1.656</td>\n",
       "      <td>-0.469</td>\n",
       "      <td>-1.437</td>\n",
       "      <td>-0.581</td>\n",
       "      <td>-0.308</td>\n",
       "      <td>-0.837</td>\n",
       "      <td>-1.739</td>\n",
       "      <td>0.037</td>\n",
       "      <td>0.336</td>\n",
       "      <td>-1.102</td>\n",
       "      <td>2.371</td>\n",
       "      <td>0.554</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.050</td>\n",
       "      <td>-0.347</td>\n",
       "      <td>0.904</td>\n",
       "      <td>-1.324</td>\n",
       "      <td>-0.849</td>\n",
       "      <td>3.432</td>\n",
       "      <td>0.222</td>\n",
       "      <td>0.416</td>\n",
       "      <td>0.174</td>\n",
       "      <td>-1.517</td>\n",
       "      <td>-0.337</td>\n",
       "      <td>0.055</td>\n",
       "      <td>-0.464</td>\n",
       "      <td>0.014</td>\n",
       "      <td>-1.073</td>\n",
       "      <td>0.325</td>\n",
       "      <td>-0.523</td>\n",
       "      <td>-0.692</td>\n",
       "      <td>0.190</td>\n",
       "      <td>-0.883</td>\n",
       "      <td>-1.830</td>\n",
       "      <td>1.408</td>\n",
       "      <td>2.319</td>\n",
       "      <td>1.704</td>\n",
       "      <td>-0.723</td>\n",
       "      <td>1.014</td>\n",
       "      <td>0.064</td>\n",
       "      <td>0.096</td>\n",
       "      <td>-0.775</td>\n",
       "      <td>1.845</td>\n",
       "      <td>0.898</td>\n",
       "      <td>0.134</td>\n",
       "      <td>2.415</td>\n",
       "      <td>-0.996</td>\n",
       "      <td>-1.006</td>\n",
       "      <td>1.378</td>\n",
       "      <td>1.246</td>\n",
       "      <td>1.478</td>\n",
       "      <td>0.428</td>\n",
       "      <td>0.253</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  target      0      1      2  ...      295    296    297    298    299\n",
       "0   0     1.0 -0.098  2.165  0.681  ...   -2.097  1.051 -0.414  1.038 -1.065\n",
       "1   1     0.0  1.081 -0.973 -0.383  ...   -1.624 -0.458 -1.099 -0.936  0.973\n",
       "2   2     1.0 -0.523 -0.089 -0.348  ...   -1.165 -1.544  0.004  0.800 -1.211\n",
       "3   3     1.0  0.067 -0.021  0.392  ...    0.467 -0.562 -0.254 -0.533  0.238\n",
       "4   4     1.0  2.347 -0.831  0.511  ...    1.378  1.246  1.478  0.428  0.253\n",
       "\n",
       "[5 rows x 302 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv('../input/train.csv')\n",
    "test = pd.read_csv('../input/test.csv')\n",
    "\n",
    "print('Train Shape: ', train.shape)\n",
    "print('Test Shape: ', test.shape)\n",
    "\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "_uuid": "0a43ca70552d6aef10acb632a26332494e5a90ec"
   },
   "outputs": [],
   "source": [
    "# prepare for modeling\n",
    "X_train = train.drop(['id', 'target'], axis=1)\n",
    "y_train = train['target']\n",
    "\n",
    "X_test = test.drop(['id'], axis=1)\n",
    "\n",
    "# scaling data\n",
    "scaler = StandardScaler()\n",
    "X_train = scaler.fit_transform(X_train)\n",
    "X_test = scaler.transform(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "4e8206b25fc1c72ac37efd45b9eb4f87781da9ef"
   },
   "source": [
    "### Baseline Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "_uuid": "9b3ba77071ced109f3488534ec03934c3b9d42c0"
   },
   "outputs": [],
   "source": [
    "# define models\n",
    "ridge = linear_model.Ridge()\n",
    "lasso = linear_model.Lasso()\n",
    "elastic = linear_model.ElasticNet()\n",
    "lasso_lars = linear_model.LassoLars()\n",
    "bayesian_ridge = linear_model.BayesianRidge()\n",
    "logistic = linear_model.LogisticRegression(solver='liblinear')\n",
    "sgd = linear_model.SGDClassifier()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "_uuid": "92ac7cade192e1dfd394d7ca4797617d2c630b0c"
   },
   "outputs": [],
   "source": [
    "models = [ridge, lasso, elastic, lasso_lars, bayesian_ridge, logistic, sgd]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "_uuid": "98130258fdfb513a43a59283f52a124cfb114a57"
   },
   "outputs": [],
   "source": [
    "# function to get cross validation scores\n",
    "def get_cv_scores(model):\n",
    "    scores = cross_val_score(model, X_train, y_train, cv=5, scoring='roc_auc')\n",
    "    print('CV Mean: ', np.mean(scores))\n",
    "    print('STD: ', np.std(scores))\n",
    "    print('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "_uuid": "f17c6ea0582b028cd7e05ec0063deecb4c2d4e05"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ridge(alpha=1.0, copy_X=True, fit_intercept=True, max_iter=None,\n",
      "   normalize=False, random_state=None, solver='auto', tol=0.001)\n",
      "CV Mean:  0.6759762475523124\n",
      "STD:  0.1170461756924883\n",
      "\n",
      "\n",
      "Lasso(alpha=1.0, copy_X=True, fit_intercept=True, max_iter=1000,\n",
      "   normalize=False, positive=False, precompute=False, random_state=None,\n",
      "   selection='cyclic', tol=0.0001, warm_start=False)\n",
      "CV Mean:  0.5\n",
      "STD:  0.0\n",
      "\n",
      "\n",
      "ElasticNet(alpha=1.0, copy_X=True, fit_intercept=True, l1_ratio=0.5,\n",
      "      max_iter=1000, normalize=False, positive=False, precompute=False,\n",
      "      random_state=None, selection='cyclic', tol=0.0001, warm_start=False)\n",
      "CV Mean:  0.5\n",
      "STD:  0.0\n",
      "\n",
      "\n",
      "LassoLars(alpha=1.0, copy_X=True, eps=2.220446049250313e-16,\n",
      "     fit_intercept=True, fit_path=True, max_iter=500, normalize=True,\n",
      "     positive=False, precompute='auto', verbose=False)\n",
      "CV Mean:  0.5\n",
      "STD:  0.0\n",
      "\n",
      "\n",
      "BayesianRidge(alpha_1=1e-06, alpha_2=1e-06, compute_score=False, copy_X=True,\n",
      "       fit_intercept=True, lambda_1=1e-06, lambda_2=1e-06, n_iter=300,\n",
      "       normalize=False, tol=0.001, verbose=False)\n",
      "CV Mean:  0.688224616492365\n",
      "STD:  0.13183095412112777\n",
      "\n",
      "\n",
      "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
      "          intercept_scaling=1, max_iter=100, multi_class='warn',\n",
      "          n_jobs=None, penalty='l2', random_state=None, solver='liblinear',\n",
      "          tol=0.0001, verbose=0, warm_start=False)\n",
      "CV Mean:  0.7447916666666667\n",
      "STD:  0.053735373404660246\n",
      "\n",
      "\n",
      "SGDClassifier(alpha=0.0001, average=False, class_weight=None,\n",
      "       early_stopping=False, epsilon=0.1, eta0=0.0, fit_intercept=True,\n",
      "       l1_ratio=0.15, learning_rate='optimal', loss='hinge', max_iter=None,\n",
      "       n_iter=None, n_iter_no_change=5, n_jobs=None, penalty='l2',\n",
      "       power_t=0.5, random_state=None, shuffle=True, tol=None,\n",
      "       validation_fraction=0.1, verbose=0, warm_start=False)\n",
      "CV Mean:  0.7333333333333333\n",
      "STD:  0.03404902964480909\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# loop through list of models\n",
    "for model in models:\n",
    "    print(model)\n",
    "    get_cv_scores(model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "7d6d3eb6e3b092858e398592e8cde0dd4d912239"
   },
   "source": [
    "From this we can see our best performing models out of the box are logistic regression and stochastic gradient descent.  Let's see if we can optimize these models with hyperparameter tuning.\n",
    "\n",
    "### Logistic Regression and Grid Search\n",
    "\n",
    "Grid search is an exhaustive search over specified parameter values."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "_uuid": "7edd0173203e4d9475ffb799a91497431dbe9844"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 128 candidates, totalling 384 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  42 tasks      | elapsed:    4.6s\n",
      "[Parallel(n_jobs=-1)]: Done 297 tasks      | elapsed:   13.5s\n",
      "[Parallel(n_jobs=-1)]: Done 377 out of 384 | elapsed:   16.8s remaining:    0.3s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Score:  0.7899186582809224\n",
      "Best Params:  {'C': 1, 'class_weight': {1: 0.6, 0: 0.4}, 'penalty': 'l1', 'solver': 'liblinear'}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done 384 out of 384 | elapsed:   17.2s finished\n"
     ]
    }
   ],
   "source": [
    "penalty = ['l1', 'l2']\n",
    "C = [0.0001, 0.001, 0.01, 0.1, 1, 10, 100, 1000]\n",
    "class_weight = [{1:0.5, 0:0.5}, {1:0.4, 0:0.6}, {1:0.6, 0:0.4}, {1:0.7, 0:0.3}]\n",
    "solver = ['liblinear', 'saga']\n",
    "\n",
    "param_grid = dict(penalty=penalty,\n",
    "                  C=C,\n",
    "                  class_weight=class_weight,\n",
    "                  solver=solver)\n",
    "\n",
    "grid = GridSearchCV(estimator=logistic, param_grid=param_grid, scoring='roc_auc', verbose=1, n_jobs=-1)\n",
    "grid_result = grid.fit(X_train, y_train)\n",
    "\n",
    "print('Best Score: ', grid_result.best_score_)\n",
    "print('Best Params: ', grid_result.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "_uuid": "8b3fbfd022c02de6e51a97130ec3c30af4cb685c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CV Mean:  0.8166666666666667\n",
      "STD:  0.0398716352734963\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "logistic = linear_model.LogisticRegression(C=1, class_weight={1:0.6, 0:0.4}, penalty='l1', solver='liblinear')\n",
    "get_cv_scores(logistic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "_uuid": "91f9287a24c6892ec1899252a6d449c1e4f5318a"
   },
   "outputs": [],
   "source": [
    "predictions = logistic.fit(X_train, y_train).predict_proba(X_test)\n",
    "#### score 0.828 on public leaderboard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "_uuid": "2c96c423b4a89591beee3765d7920d5ce491d6f3"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>250</td>\n",
       "      <td>0.232936</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>251</td>\n",
       "      <td>0.529461</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>252</td>\n",
       "      <td>0.222784</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>253</td>\n",
       "      <td>0.001155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>254</td>\n",
       "      <td>0.132049</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    id    target\n",
       "0  250  0.232936\n",
       "1  251  0.529461\n",
       "2  252  0.222784\n",
       "3  253  0.001155\n",
       "4  254  0.132049"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submission = pd.read_csv('../input/sample_submission.csv')\n",
    "submission['target'] = predictions\n",
    "#submission.to_csv('submission.csv', index=False)\n",
    "submission.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "b62c8810bb88c70413d502cc1cf60eb0c117ddfe"
   },
   "source": [
    "### Stochastic Gradient Descent and Random Search\n",
    "\n",
    "Random search is a random (obviously) search over specified parameter values."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "_uuid": "f450d2823cc524633b5344a07fb8e7e7e18df91a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 1000 candidates, totalling 3000 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done 448 tasks      | elapsed:    5.5s\n",
      "[Parallel(n_jobs=-1)]: Done 1948 tasks      | elapsed:   23.8s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Score:  0.7980969951083158\n",
      "Best Params:  {'penalty': 'elasticnet', 'loss': 'log', 'learning_rate': 'invscaling', 'eta0': 1, 'class_weight': {1: 0.7, 0: 0.3}, 'alpha': 0.1}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done 3000 out of 3000 | elapsed:   35.8s finished\n"
     ]
    }
   ],
   "source": [
    "loss = ['hinge', 'log', 'modified_huber', 'squared_hinge', 'perceptron']\n",
    "penalty = ['l1', 'l2', 'elasticnet']\n",
    "alpha = [0.0001, 0.001, 0.01, 0.1, 1, 10, 100, 1000]\n",
    "learning_rate = ['constant', 'optimal', 'invscaling', 'adaptive']\n",
    "class_weight = [{1:0.5, 0:0.5}, {1:0.4, 0:0.6}, {1:0.6, 0:0.4}, {1:0.7, 0:0.3}]\n",
    "eta0 = [1, 10, 100]\n",
    "\n",
    "param_distributions = dict(loss=loss,\n",
    "                           penalty=penalty,\n",
    "                           alpha=alpha,\n",
    "                           learning_rate=learning_rate,\n",
    "                           class_weight=class_weight,\n",
    "                           eta0=eta0)\n",
    "\n",
    "random = RandomizedSearchCV(estimator=sgd, param_distributions=param_distributions, scoring='roc_auc', verbose=1, n_jobs=-1, n_iter=1000)\n",
    "random_result = random.fit(X_train, y_train)\n",
    "\n",
    "print('Best Score: ', random_result.best_score_)\n",
    "print('Best Params: ', random_result.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "_uuid": "4f2bde1a0b3435d15e849356ed93784d02b30011"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CV Mean:  0.7826388888888889\n",
      "STD:  0.05031009167228856\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "sgd = linear_model.SGDClassifier(alpha=0.1,\n",
    "                                 class_weight={1:0.7, 0:0.3},\n",
    "                                 eta0=100,\n",
    "                                 learning_rate='optimal',\n",
    "                                 loss='log',\n",
    "                                 penalty='elasticnet')\n",
    "get_cv_scores(sgd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "_uuid": "b6431fcc35280c4863804af277cee785dd7b046d"
   },
   "outputs": [],
   "source": [
    "predictions = sgd.fit(X_train, y_train).predict_proba(X_test)\n",
    "#### score 0.790 on public leaderboard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "_uuid": "21cdfaec4875540e77cf8aa337e7e5d80cc1b4a4"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>250</td>\n",
       "      <td>0.136742</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>251</td>\n",
       "      <td>0.147602</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>252</td>\n",
       "      <td>0.164560</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>253</td>\n",
       "      <td>0.107634</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>254</td>\n",
       "      <td>0.164928</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    id    target\n",
       "0  250  0.136742\n",
       "1  251  0.147602\n",
       "2  252  0.164560\n",
       "3  253  0.107634\n",
       "4  254  0.164928"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submission = pd.read_csv('../input/sample_submission.csv')\n",
    "submission['target'] = predictions\n",
    "submission.to_csv('submission.csv', index=False)\n",
    "submission.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "_uuid": "1952c3f1f88f33552880844f4a8c69e949b4e682"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}