notebooks/mgbm_hmda.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# License\n",
    "\n",
    "Copyright 2019 Navdeep Gill, Patrick Hall, Kim Montgomery, Nick Schmidt\n",
    "\n",
    "Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at\n",
    "\n",
    "http://www.apache.org/licenses/LICENSE-2.0\n",
    "\n",
    "Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.\n",
    "\n",
    "**DISCLAIMER**: This notebook is not legal compliance advice."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import Libraries and Initialize `h2o` Cluster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checking whether there is an H2O instance running at http://localhost:54321 . connected.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td>H2O cluster uptime:</td>\n",
       "<td>09 secs</td></tr>\n",
       "<tr><td>H2O cluster timezone:</td>\n",
       "<td>America/Los_Angeles</td></tr>\n",
       "<tr><td>H2O data parsing timezone:</td>\n",
       "<td>UTC</td></tr>\n",
       "<tr><td>H2O cluster version:</td>\n",
       "<td>3.26.0.9</td></tr>\n",
       "<tr><td>H2O cluster version age:</td>\n",
       "<td>1 month and 9 days </td></tr>\n",
       "<tr><td>H2O cluster name:</td>\n",
       "<td>H2O_from_python_navdeep_vcnglh</td></tr>\n",
       "<tr><td>H2O cluster total nodes:</td>\n",
       "<td>1</td></tr>\n",
       "<tr><td>H2O cluster free memory:</td>\n",
       "<td>28.14 Gb</td></tr>\n",
       "<tr><td>H2O cluster total cores:</td>\n",
       "<td>40</td></tr>\n",
       "<tr><td>H2O cluster allowed cores:</td>\n",
       "<td>40</td></tr>\n",
       "<tr><td>H2O cluster status:</td>\n",
       "<td>locked, healthy</td></tr>\n",
       "<tr><td>H2O connection url:</td>\n",
       "<td>http://localhost:54321</td></tr>\n",
       "<tr><td>H2O connection proxy:</td>\n",
       "<td>{'http': None, 'https': None}</td></tr>\n",
       "<tr><td>H2O internal security:</td>\n",
       "<td>False</td></tr>\n",
       "<tr><td>H2O API Extensions:</td>\n",
       "<td>Amazon S3, XGBoost, Algos, AutoML, Core V3, TargetEncoder, Core V4</td></tr>\n",
       "<tr><td>Python version:</td>\n",
       "<td>3.6.6 final</td></tr></table></div>"
      ],
      "text/plain": [
       "--------------------------  ------------------------------------------------------------------\n",
       "H2O cluster uptime:         09 secs\n",
       "H2O cluster timezone:       America/Los_Angeles\n",
       "H2O data parsing timezone:  UTC\n",
       "H2O cluster version:        3.26.0.9\n",
       "H2O cluster version age:    1 month and 9 days\n",
       "H2O cluster name:           H2O_from_python_navdeep_vcnglh\n",
       "H2O cluster total nodes:    1\n",
       "H2O cluster free memory:    28.14 Gb\n",
       "H2O cluster total cores:    40\n",
       "H2O cluster allowed cores:  40\n",
       "H2O cluster status:         locked, healthy\n",
       "H2O connection url:         http://localhost:54321\n",
       "H2O connection proxy:       {'http': None, 'https': None}\n",
       "H2O internal security:      False\n",
       "H2O API Extensions:         Amazon S3, XGBoost, Algos, AutoML, Core V3, TargetEncoder, Core V4\n",
       "Python version:             3.6.6 final\n",
       "--------------------------  ------------------------------------------------------------------"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import h2o\n",
    "import os\n",
    "import numpy as np\n",
    "import time\n",
    "import math\n",
    "import matplotlib.pyplot as plt\n",
    "import shap\n",
    "import pandas as pd\n",
    "\n",
    "from h2o.estimators.gbm import H2OGradientBoostingEstimator\n",
    "from h2o.estimators.xgboost import H2OXGBoostEstimator\n",
    "from h2o.grid.grid_search import H2OGridSearch \n",
    "\n",
    "# init h2o\n",
    "h2o.init(min_mem_size='30G')\n",
    "h2o.remove_all()\n",
    "\n",
    "# set random seed for reproducibility\n",
    "SEED = 22222"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import Processed HMDA Train and Test Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parse progress: |█████████████████████████████████████████████████████████| 100%\n",
      "Parse progress: |█████████████████████████████████████████████████████████| 100%\n"
     ]
    }
   ],
   "source": [
    "train_path = os.sep.join(['..', 'data', 'output', 'hmda_train_processed.csv'])\n",
    "train = h2o.import_file(train_path)\n",
    "\n",
    "test_path = os.sep.join(['..', 'data', 'output', 'hmda_test_processed.csv'])\n",
    "test = h2o.import_file(test_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train shape: (160338, 24)\n",
      "Test shape: (39662, 23)\n"
     ]
    }
   ],
   "source": [
    "print(f\"Train shape: {train.shape}\")\n",
    "print(f\"Test shape: {test.shape}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead>\n",
       "<tr><th style=\"text-align: right;\">      Unnamed: 0</th><th style=\"text-align: right;\">  high_priced</th><th style=\"text-align: right;\">  term_360</th><th style=\"text-align: right;\">  conforming</th><th style=\"text-align: right;\">  black</th><th style=\"text-align: right;\">  asian</th><th style=\"text-align: right;\">  white</th><th style=\"text-align: right;\">  amind</th><th style=\"text-align: right;\">  hipac</th><th style=\"text-align: right;\">  hispanic</th><th style=\"text-align: right;\">  non_hispanic</th><th style=\"text-align: right;\">  male</th><th style=\"text-align: right;\">  female</th><th style=\"text-align: right;\">  agegte62</th><th style=\"text-align: right;\">  agelt62</th><th style=\"text-align: right;\">  cv_fold</th><th>debt_to_income_ratio_missing  </th><th style=\"text-align: right;\">  loan_amount_std</th><th style=\"text-align: right;\">  loan_to_value_ratio_std</th><th style=\"text-align: right;\">  no_intro_rate_period_std</th><th style=\"text-align: right;\">  intro_rate_period_std</th><th style=\"text-align: right;\">  property_value_std</th><th style=\"text-align: right;\">  income_std</th><th style=\"text-align: right;\">  debt_to_income_ratio_std</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td style=\"text-align: right;\">     2.5493e+06 </td><td style=\"text-align: right;\">            0</td><td style=\"text-align: right;\">         1</td><td style=\"text-align: right;\">           1</td><td style=\"text-align: right;\">    nan</td><td style=\"text-align: right;\">    nan</td><td style=\"text-align: right;\">    nan</td><td style=\"text-align: right;\">    nan</td><td style=\"text-align: right;\">    nan</td><td style=\"text-align: right;\">       nan</td><td style=\"text-align: right;\">           nan</td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">       0</td><td style=\"text-align: right;\">         1</td><td style=\"text-align: right;\">        0</td><td style=\"text-align: right;\">        1</td><td>False                         </td><td style=\"text-align: right;\">       -0.514393 </td><td style=\"text-align: right;\">                 0.333922</td><td style=\"text-align: right;\">                  0.244394</td><td style=\"text-align: right;\">              -0.215304</td><td style=\"text-align: right;\">           -0.535932</td><td style=\"text-align: right;\">  -0.0403074</td><td style=\"text-align: right;\">                 0.854601 </td></tr>\n",
       "<tr><td style=\"text-align: right;\">     4.00076e+06</td><td style=\"text-align: right;\">            0</td><td style=\"text-align: right;\">         1</td><td style=\"text-align: right;\">           1</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">      1</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">         0</td><td style=\"text-align: right;\">             1</td><td style=\"text-align: right;\">   nan</td><td style=\"text-align: right;\">     nan</td><td style=\"text-align: right;\">         1</td><td style=\"text-align: right;\">        0</td><td style=\"text-align: right;\">        5</td><td>False                         </td><td style=\"text-align: right;\">       -0.118642 </td><td style=\"text-align: right;\">                 0.268727</td><td style=\"text-align: right;\">                  0.244394</td><td style=\"text-align: right;\">              -0.215304</td><td style=\"text-align: right;\">           -0.227585</td><td style=\"text-align: right;\">  -0.0181326</td><td style=\"text-align: right;\">                -0.425131 </td></tr>\n",
       "<tr><td style=\"text-align: right;\">     1.54693e+06</td><td style=\"text-align: right;\">            0</td><td style=\"text-align: right;\">         1</td><td style=\"text-align: right;\">           1</td><td style=\"text-align: right;\">    nan</td><td style=\"text-align: right;\">    nan</td><td style=\"text-align: right;\">    nan</td><td style=\"text-align: right;\">    nan</td><td style=\"text-align: right;\">    nan</td><td style=\"text-align: right;\">       nan</td><td style=\"text-align: right;\">           nan</td><td style=\"text-align: right;\">   nan</td><td style=\"text-align: right;\">     nan</td><td style=\"text-align: right;\">         1</td><td style=\"text-align: right;\">        0</td><td style=\"text-align: right;\">        5</td><td>False                         </td><td style=\"text-align: right;\">       -0.778227 </td><td style=\"text-align: right;\">                 0.228996</td><td style=\"text-align: right;\">                 -4.09175 </td><td style=\"text-align: right;\">               4.61086 </td><td style=\"text-align: right;\">           -0.720941</td><td style=\"text-align: right;\">  -0.0323383</td><td style=\"text-align: right;\">                 0.123326 </td></tr>\n",
       "<tr><td style=\"text-align: right;\">     5.45314e+06</td><td style=\"text-align: right;\">            0</td><td style=\"text-align: right;\">         1</td><td style=\"text-align: right;\">           1</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">      1</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">         0</td><td style=\"text-align: right;\">             1</td><td style=\"text-align: right;\">   nan</td><td style=\"text-align: right;\">     nan</td><td style=\"text-align: right;\">         0</td><td style=\"text-align: right;\">        1</td><td style=\"text-align: right;\">        3</td><td>False                         </td><td style=\"text-align: right;\">       -0.0746695</td><td style=\"text-align: right;\">                -1.15024 </td><td style=\"text-align: right;\">                  0.244394</td><td style=\"text-align: right;\">              -0.215304</td><td style=\"text-align: right;\">            0.358276</td><td style=\"text-align: right;\">  -0.0181326</td><td style=\"text-align: right;\">                -0.425131 </td></tr>\n",
       "<tr><td style=\"text-align: right;\">     4.94313e+06</td><td style=\"text-align: right;\">            0</td><td style=\"text-align: right;\">         1</td><td style=\"text-align: right;\">           1</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">      1</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">         1</td><td style=\"text-align: right;\">             0</td><td style=\"text-align: right;\">   nan</td><td style=\"text-align: right;\">     nan</td><td style=\"text-align: right;\">         0</td><td style=\"text-align: right;\">        1</td><td style=\"text-align: right;\">        4</td><td>False                         </td><td style=\"text-align: right;\">       -0.602338 </td><td style=\"text-align: right;\">                 0.55252 </td><td style=\"text-align: right;\">                  0.244394</td><td style=\"text-align: right;\">              -0.215304</td><td style=\"text-align: right;\">           -0.628437</td><td style=\"text-align: right;\">  -0.0382285</td><td style=\"text-align: right;\">                 0.763191 </td></tr>\n",
       "<tr><td style=\"text-align: right;\">     1.23729e+06</td><td style=\"text-align: right;\">            0</td><td style=\"text-align: right;\">         1</td><td style=\"text-align: right;\">           1</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">      1</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">         0</td><td style=\"text-align: right;\">             1</td><td style=\"text-align: right;\">   nan</td><td style=\"text-align: right;\">     nan</td><td style=\"text-align: right;\">         0</td><td style=\"text-align: right;\">        1</td><td style=\"text-align: right;\">        3</td><td>False                         </td><td style=\"text-align: right;\">       -0.338504 </td><td style=\"text-align: right;\">                 0.372784</td><td style=\"text-align: right;\">                  0.244394</td><td style=\"text-align: right;\">              -0.215304</td><td style=\"text-align: right;\">           -0.412593</td><td style=\"text-align: right;\">  -0.0247158</td><td style=\"text-align: right;\">                 0.397554 </td></tr>\n",
       "<tr><td style=\"text-align: right;\">682016          </td><td style=\"text-align: right;\">            0</td><td style=\"text-align: right;\">         1</td><td style=\"text-align: right;\">           1</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">      1</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">         0</td><td style=\"text-align: right;\">             1</td><td style=\"text-align: right;\">     0</td><td style=\"text-align: right;\">       1</td><td style=\"text-align: right;\">       nan</td><td style=\"text-align: right;\">      nan</td><td style=\"text-align: right;\">        1</td><td>False                         </td><td style=\"text-align: right;\">       -0.426448 </td><td style=\"text-align: right;\">                 0.355249</td><td style=\"text-align: right;\">                  0.244394</td><td style=\"text-align: right;\">              -0.215304</td><td style=\"text-align: right;\">           -0.474263</td><td style=\"text-align: right;\">  -0.0209045</td><td style=\"text-align: right;\">                 1.03742  </td></tr>\n",
       "<tr><td style=\"text-align: right;\"> 19740          </td><td style=\"text-align: right;\">            0</td><td style=\"text-align: right;\">         1</td><td style=\"text-align: right;\">           1</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">      1</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">         0</td><td style=\"text-align: right;\">             1</td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">       0</td><td style=\"text-align: right;\">       nan</td><td style=\"text-align: right;\">      nan</td><td style=\"text-align: right;\">        2</td><td>False                         </td><td style=\"text-align: right;\">       -0.734255 </td><td style=\"text-align: right;\">                 0.55252 </td><td style=\"text-align: right;\">                  0.244394</td><td style=\"text-align: right;\">              -0.215304</td><td style=\"text-align: right;\">           -0.720941</td><td style=\"text-align: right;\">  -0.0396144</td><td style=\"text-align: right;\">                 0.488963 </td></tr>\n",
       "<tr><td style=\"text-align: right;\">     5.36388e+06</td><td style=\"text-align: right;\">            0</td><td style=\"text-align: right;\">         1</td><td style=\"text-align: right;\">           1</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">      1</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">         0</td><td style=\"text-align: right;\">             1</td><td style=\"text-align: right;\">   nan</td><td style=\"text-align: right;\">     nan</td><td style=\"text-align: right;\">         0</td><td style=\"text-align: right;\">        1</td><td style=\"text-align: right;\">        5</td><td>False                         </td><td style=\"text-align: right;\">        0.277109 </td><td style=\"text-align: right;\">                 0.142995</td><td style=\"text-align: right;\">                  0.244394</td><td style=\"text-align: right;\">              -0.215304</td><td style=\"text-align: right;\">            0.111598</td><td style=\"text-align: right;\">  -0.019865 </td><td style=\"text-align: right;\">                 0.0319161</td></tr>\n",
       "<tr><td style=\"text-align: right;\">     1.94229e+06</td><td style=\"text-align: right;\">            0</td><td style=\"text-align: right;\">         1</td><td style=\"text-align: right;\">           0</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">      1</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">         0</td><td style=\"text-align: right;\">             1</td><td style=\"text-align: right;\">   nan</td><td style=\"text-align: right;\">     nan</td><td style=\"text-align: right;\">         0</td><td style=\"text-align: right;\">        1</td><td style=\"text-align: right;\">        4</td><td>False                         </td><td style=\"text-align: right;\">        1.33245  </td><td style=\"text-align: right;\">                -0.261719</td><td style=\"text-align: right;\">                  0.244394</td><td style=\"text-align: right;\">              -0.215304</td><td style=\"text-align: right;\">            1.19082 </td><td style=\"text-align: right;\">   0.0757636</td><td style=\"text-align: right;\">                -1.15641  </td></tr>\n",
       "</tbody>\n",
       "</table>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": []
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead>\n",
       "<tr><th>       </th><th>Unnamed: 0        </th><th>high_priced        </th><th>term_360          </th><th>conforming         </th><th>black             </th><th>asian              </th><th>white             </th><th>amind               </th><th>hipac                </th><th>hispanic           </th><th>non_hispanic      </th><th>male               </th><th>female             </th><th>agegte62           </th><th>agelt62           </th><th>cv_fold          </th><th>debt_to_income_ratio_missing  </th><th>loan_amount_std      </th><th>loan_to_value_ratio_std  </th><th>no_intro_rate_period_std  </th><th>intro_rate_period_std  </th><th>property_value_std    </th><th>income_std           </th><th>debt_to_income_ratio_std  </th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>type   </td><td>int               </td><td>int                </td><td>int               </td><td>int                </td><td>int               </td><td>int                </td><td>int               </td><td>int                 </td><td>int                  </td><td>int                </td><td>int               </td><td>int                </td><td>int                </td><td>int                </td><td>int               </td><td>int              </td><td>enum                          </td><td>real                 </td><td>real                     </td><td>real                      </td><td>real                   </td><td>real                  </td><td>real                 </td><td>real                      </td></tr>\n",
       "<tr><td>mins   </td><td>49.0              </td><td>0.0                </td><td>0.0               </td><td>0.0                </td><td>0.0               </td><td>0.0                </td><td>0.0               </td><td>0.0                 </td><td>0.0                  </td><td>0.0                </td><td>0.0               </td><td>0.0                </td><td>0.0                </td><td>0.0                </td><td>0.0               </td><td>1.0              </td><td>                              </td><td>-1.26192360553821    </td><td>-3.4792433327941232      </td><td>-4.09174686250922         </td><td>-0.21530422241413488   </td><td>-1.0909583443261746   </td><td>-0.09782313461197675 </td><td>-2.527546647809547        </td></tr>\n",
       "<tr><td>mean   </td><td>3319386.4112125635</td><td>0.09670196709451284</td><td>0.9603275580336539</td><td>0.928027042871933  </td><td>0.0787575295739894</td><td>0.07867769794614994</td><td>0.8355541040714131</td><td>0.004434284055446694</td><td>0.0025763843530009433</td><td>0.12107167983057363</td><td>0.8789283201694263</td><td>0.6126165686353815 </td><td>0.3873834313646186 </td><td>0.21633871324342252</td><td>0.7836612867565775</td><td>3.003037333632701</td><td>                              </td><td>8.938396999015344e-17</td><td>-3.775663977769495e-17   </td><td>1.3400948202787506e-16    </td><td>-4.254269270726192e-18 </td><td>-1.063567317681548e-18</td><td>0.0                  </td><td>-2.7510941284029374e-16   </td></tr>\n",
       "<tr><td>maxs   </td><td>6504996.0         </td><td>1.0                </td><td>1.0               </td><td>1.0                </td><td>1.0               </td><td>1.0                </td><td>1.0               </td><td>1.0                 </td><td>1.0                  </td><td>1.0                </td><td>1.0               </td><td>1.0                </td><td>1.0                </td><td>1.0                </td><td>1.0               </td><td>5.0              </td><td>                              </td><td>51.50492551541175    </td><td>202.75528359110467       </td><td>0.24439439525512616       </td><td>16.877348931294414     </td><td>40.8443375990408      </td><td>305.5429652264891    </td><td>3.87111034635691          </td></tr>\n",
       "<tr><td>sigma  </td><td>1952820.8522752034</td><td>0.295552434347859  </td><td>0.1951890799460008</td><td>0.25844393424533546</td><td>0.2693609245488   </td><td>0.2692360374620666 </td><td>0.3706810494787445</td><td>0.06644285679708556 </td><td>0.05069285202604093  </td><td>0.3262117410182472 </td><td>0.3262117410182472</td><td>0.48715526363003847</td><td>0.48715526363003847</td><td>0.4117503573990468 </td><td>0.4117503573990468</td><td>1.41151976175012 </td><td>                              </td><td>1.0000031184269407   </td><td>1.0000031184269407       </td><td>1.0000031184269407        </td><td>1.0000031184269407     </td><td>1.0000031184269405    </td><td>1.000003118426941    </td><td>1.0000031184269407        </td></tr>\n",
       "<tr><td>zeros  </td><td>0                 </td><td>144833             </td><td>6361              </td><td>11540              </td><td>126938            </td><td>126949             </td><td>22659             </td><td>137179              </td><td>137435               </td><td>121184             </td><td>16693             </td><td>33523              </td><td>53014              </td><td>63801              </td><td>17613             </td><td>0                </td><td>                              </td><td>0                    </td><td>0                        </td><td>0                         </td><td>0                      </td><td>0                     </td><td>0                    </td><td>0                         </td></tr>\n",
       "<tr><td>missing</td><td>0                 </td><td>0                  </td><td>0                 </td><td>0                  </td><td>22548             </td><td>22548              </td><td>22548             </td><td>22548               </td><td>22548                </td><td>22461              </td><td>22461             </td><td>73801              </td><td>73801              </td><td>78924              </td><td>78924             </td><td>0                </td><td>0                             </td><td>0                    </td><td>0                        </td><td>0                         </td><td>0                      </td><td>0                     </td><td>0                    </td><td>0                         </td></tr>\n",
       "<tr><td>0      </td><td>2549300.0         </td><td>0.0                </td><td>1.0               </td><td>1.0                </td><td>nan               </td><td>nan                </td><td>nan               </td><td>nan                 </td><td>nan                  </td><td>nan                </td><td>nan               </td><td>1.0                </td><td>0.0                </td><td>1.0                </td><td>0.0               </td><td>1.0              </td><td>False                         </td><td>-0.5143932429914191  </td><td>0.3339223958668174       </td><td>0.24439439525512616       </td><td>-0.21530422241413488   </td><td>-0.5359323686051412   </td><td>-0.04030738198147695 </td><td>0.8546006205355803        </td></tr>\n",
       "<tr><td>1      </td><td>4000757.0         </td><td>0.0                </td><td>1.0               </td><td>1.0                </td><td>0.0               </td><td>0.0                </td><td>1.0               </td><td>0.0                 </td><td>0.0                  </td><td>0.0                </td><td>1.0               </td><td>nan                </td><td>nan                </td><td>1.0                </td><td>0.0               </td><td>5.0              </td><td>False                         </td><td>-0.11864187458429434 </td><td>0.26872662571185846      </td><td>0.24439439525512616       </td><td>-0.21530422241413488   </td><td>-0.22758460431567815  </td><td>-0.018132633979356542</td><td>-0.4251307782977112       </td></tr>\n",
       "<tr><td>2      </td><td>1546928.0         </td><td>0.0                </td><td>1.0               </td><td>1.0                </td><td>nan               </td><td>nan                </td><td>nan               </td><td>nan                 </td><td>nan                  </td><td>nan                </td><td>nan               </td><td>nan                </td><td>nan                </td><td>1.0                </td><td>0.0               </td><td>5.0              </td><td>False                         </td><td>-0.7782274885961689  </td><td>0.228995556370366        </td><td>-4.09174686250922         </td><td>4.610856668044749      </td><td>-0.720941027178819    </td><td>-0.03233833191821493 </td><td>0.12332553548798514       </td></tr>\n",
       "<tr><td>3      </td><td>5453145.0         </td><td>0.0                </td><td>1.0               </td><td>1.0                </td><td>0.0               </td><td>0.0                </td><td>1.0               </td><td>0.0                 </td><td>0.0                  </td><td>0.0                </td><td>1.0               </td><td>nan                </td><td>nan                </td><td>0.0                </td><td>1.0               </td><td>3.0              </td><td>False                         </td><td>-0.07466950031683604 </td><td>-1.1502401364843071      </td><td>0.24439439525512616       </td><td>-0.21530422241413488   </td><td>0.3582761478343016    </td><td>-0.018132633979356542</td><td>-0.4251307782977112       </td></tr>\n",
       "<tr><td>4      </td><td>4943130.0         </td><td>0.0                </td><td>1.0               </td><td>1.0                </td><td>0.0               </td><td>0.0                </td><td>1.0               </td><td>0.0                 </td><td>0.0                  </td><td>1.0                </td><td>0.0               </td><td>nan                </td><td>nan                </td><td>0.0                </td><td>1.0               </td><td>4.0              </td><td>False                         </td><td>-0.6023379915263356  </td><td>0.5525199781510916       </td><td>0.24439439525512616       </td><td>-0.21530422241413488   </td><td>-0.6284366978919801   </td><td>-0.038228499356278166</td><td>0.7631912349046309        </td></tr>\n",
       "<tr><td>5      </td><td>1237293.0         </td><td>0.0                </td><td>1.0               </td><td>1.0                </td><td>0.0               </td><td>0.0                </td><td>1.0               </td><td>0.0                 </td><td>0.0                  </td><td>0.0                </td><td>1.0               </td><td>nan                </td><td>nan                </td><td>0.0                </td><td>1.0               </td><td>3.0              </td><td>False                         </td><td>-0.33850374592158583 </td><td>0.3727841882729108       </td><td>0.24439439525512616       </td><td>-0.21530422241413488   </td><td>-0.412593262889356    </td><td>-0.02471576229248604 </td><td>0.3975536923808333        </td></tr>\n",
       "<tr><td>6      </td><td>682016.0          </td><td>0.0                </td><td>1.0               </td><td>1.0                </td><td>0.0               </td><td>0.0                </td><td>1.0               </td><td>0.0                 </td><td>0.0                  </td><td>0.0                </td><td>1.0               </td><td>0.0                </td><td>1.0                </td><td>nan                </td><td>nan               </td><td>1.0              </td><td>False                         </td><td>-0.42644849445650246 </td><td>0.355248989260405        </td><td>0.24439439525512616       </td><td>-0.21530422241413488   </td><td>-0.4742628157472486   </td><td>-0.020904477479621597</td><td>1.037419391797479         </td></tr>\n",
       "<tr><td>7      </td><td>19740.0           </td><td>0.0                </td><td>1.0               </td><td>1.0                </td><td>0.0               </td><td>0.0                </td><td>1.0               </td><td>0.0                 </td><td>0.0                  </td><td>0.0                </td><td>1.0               </td><td>1.0                </td><td>0.0                </td><td>nan                </td><td>nan               </td><td>2.0              </td><td>False                         </td><td>-0.7342551143287105  </td><td>0.5525199781510916       </td><td>0.24439439525512616       </td><td>-0.21530422241413488   </td><td>-0.720941027178819    </td><td>-0.03961442110641069 </td><td>0.4889630780117827        </td></tr>\n",
       "<tr><td>8      </td><td>5363880.0         </td><td>0.0                </td><td>1.0               </td><td>1.0                </td><td>0.0               </td><td>0.0                </td><td>1.0               </td><td>0.0                 </td><td>0.0                  </td><td>0.0                </td><td>1.0               </td><td>nan                </td><td>nan                </td><td>0.0                </td><td>1.0               </td><td>5.0              </td><td>False                         </td><td>0.27710949382283034  </td><td>0.1429953936185274       </td><td>0.24439439525512616       </td><td>-0.21530422241413488   </td><td>0.1115979364027312    </td><td>-0.019865036167022204</td><td>0.03191614985703575       </td></tr>\n",
       "<tr><td>9      </td><td>1942292.0         </td><td>0.0                </td><td>1.0               </td><td>0.0                </td><td>0.0               </td><td>0.0                </td><td>1.0               </td><td>0.0                 </td><td>0.0                  </td><td>0.0                </td><td>1.0               </td><td>nan                </td><td>nan                </td><td>0.0                </td><td>1.0               </td><td>4.0              </td><td>False                         </td><td>1.3324464762418295   </td><td>-0.26171933841113787     </td><td>0.24439439525512616       </td><td>-0.21530422241413488   </td><td>1.1908151114158518    </td><td>0.07576356459212204  </td><td>-1.1564058633453065       </td></tr>\n",
       "</tbody>\n",
       "</table>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead>\n",
       "<tr><th style=\"text-align: right;\">      Unnamed: 0</th><th style=\"text-align: right;\">  high_priced</th><th style=\"text-align: right;\">  term_360</th><th style=\"text-align: right;\">  conforming</th><th style=\"text-align: right;\">  black</th><th style=\"text-align: right;\">  asian</th><th style=\"text-align: right;\">  white</th><th style=\"text-align: right;\">  amind</th><th style=\"text-align: right;\">  hipac</th><th style=\"text-align: right;\">  hispanic</th><th style=\"text-align: right;\">  non_hispanic</th><th style=\"text-align: right;\">  male</th><th style=\"text-align: right;\">  female</th><th style=\"text-align: right;\">  agegte62</th><th style=\"text-align: right;\">  agelt62</th><th>debt_to_income_ratio_missing  </th><th style=\"text-align: right;\">  loan_amount_std</th><th style=\"text-align: right;\">  loan_to_value_ratio_std</th><th style=\"text-align: right;\">  no_intro_rate_period_std</th><th style=\"text-align: right;\">  intro_rate_period_std</th><th style=\"text-align: right;\">  property_value_std</th><th style=\"text-align: right;\">  income_std</th><th style=\"text-align: right;\">  debt_to_income_ratio_std</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td style=\"text-align: right;\">131497          </td><td style=\"text-align: right;\">            0</td><td style=\"text-align: right;\">         1</td><td style=\"text-align: right;\">           1</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">      1</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">         1</td><td style=\"text-align: right;\">             0</td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">       0</td><td style=\"text-align: right;\">       nan</td><td style=\"text-align: right;\">      nan</td><td>False                         </td><td style=\"text-align: right;\">        -0.470421</td><td style=\"text-align: right;\">                 0.345133</td><td style=\"text-align: right;\">                  0.244394</td><td style=\"text-align: right;\">              -0.215304</td><td style=\"text-align: right;\">          -0.505098 </td><td style=\"text-align: right;\"> -0.0080847 </td><td style=\"text-align: right;\">                -2.52755  </td></tr>\n",
       "<tr><td style=\"text-align: right;\">851117          </td><td style=\"text-align: right;\">            0</td><td style=\"text-align: right;\">         1</td><td style=\"text-align: right;\">           1</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">      1</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">         0</td><td style=\"text-align: right;\">             1</td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">       0</td><td style=\"text-align: right;\">       nan</td><td style=\"text-align: right;\">      nan</td><td>False                         </td><td style=\"text-align: right;\">         0.321082</td><td style=\"text-align: right;\">                 0.342439</td><td style=\"text-align: right;\">                 -4.09175 </td><td style=\"text-align: right;\">               3.20323 </td><td style=\"text-align: right;\">           0.0807632</td><td style=\"text-align: right;\"> -0.0108565 </td><td style=\"text-align: right;\">                -1.15641  </td></tr>\n",
       "<tr><td style=\"text-align: right;\">835616          </td><td style=\"text-align: right;\">            0</td><td style=\"text-align: right;\">         1</td><td style=\"text-align: right;\">           1</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">      1</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">         0</td><td style=\"text-align: right;\">             1</td><td style=\"text-align: right;\">   nan</td><td style=\"text-align: right;\">     nan</td><td style=\"text-align: right;\">         0</td><td style=\"text-align: right;\">        1</td><td>False                         </td><td style=\"text-align: right;\">        -0.118642</td><td style=\"text-align: right;\">                 0.55252 </td><td style=\"text-align: right;\">                  0.244394</td><td style=\"text-align: right;\">              -0.215304</td><td style=\"text-align: right;\">          -0.289254 </td><td style=\"text-align: right;\"> -0.0160538 </td><td style=\"text-align: right;\">                 1.03742  </td></tr>\n",
       "<tr><td style=\"text-align: right;\">121491          </td><td style=\"text-align: right;\">            0</td><td style=\"text-align: right;\">         0</td><td style=\"text-align: right;\">           1</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">      1</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">         0</td><td style=\"text-align: right;\">             1</td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">       0</td><td style=\"text-align: right;\">         1</td><td style=\"text-align: right;\">        0</td><td>False                         </td><td style=\"text-align: right;\">        -0.64631 </td><td style=\"text-align: right;\">                -1.19197 </td><td style=\"text-align: right;\">                  0.244394</td><td style=\"text-align: right;\">              -0.215304</td><td style=\"text-align: right;\">          -0.320089 </td><td style=\"text-align: right;\"> -0.02922   </td><td style=\"text-align: right;\">                 0.0319161</td></tr>\n",
       "<tr><td style=\"text-align: right;\">     4.47964e+06</td><td style=\"text-align: right;\">            0</td><td style=\"text-align: right;\">         1</td><td style=\"text-align: right;\">           1</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">      1</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">         0</td><td style=\"text-align: right;\">             1</td><td style=\"text-align: right;\">   nan</td><td style=\"text-align: right;\">     nan</td><td style=\"text-align: right;\">         0</td><td style=\"text-align: right;\">        1</td><td>False                         </td><td style=\"text-align: right;\">         0.277109</td><td style=\"text-align: right;\">                 0.336837</td><td style=\"text-align: right;\">                  0.244394</td><td style=\"text-align: right;\">              -0.215304</td><td style=\"text-align: right;\">           0.0499284</td><td style=\"text-align: right;\">  0.00057731</td><td style=\"text-align: right;\">                 0.763191 </td></tr>\n",
       "<tr><td style=\"text-align: right;\">843194          </td><td style=\"text-align: right;\">            1</td><td style=\"text-align: right;\">         1</td><td style=\"text-align: right;\">           1</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">      1</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">       nan</td><td style=\"text-align: right;\">           nan</td><td style=\"text-align: right;\">   nan</td><td style=\"text-align: right;\">     nan</td><td style=\"text-align: right;\">         0</td><td style=\"text-align: right;\">        1</td><td>False                         </td><td style=\"text-align: right;\">         0.321082</td><td style=\"text-align: right;\">                 0.444679</td><td style=\"text-align: right;\">                  0.244394</td><td style=\"text-align: right;\">              -0.215304</td><td style=\"text-align: right;\">           0.0499284</td><td style=\"text-align: right;\"> -0.0351102 </td><td style=\"text-align: right;\">                 3.87111  </td></tr>\n",
       "<tr><td style=\"text-align: right;\">     6.37726e+06</td><td style=\"text-align: right;\">            0</td><td style=\"text-align: right;\">         1</td><td style=\"text-align: right;\">           1</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">      1</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">         0</td><td style=\"text-align: right;\">             1</td><td style=\"text-align: right;\">     0</td><td style=\"text-align: right;\">       1</td><td style=\"text-align: right;\">       nan</td><td style=\"text-align: right;\">      nan</td><td>False                         </td><td style=\"text-align: right;\">        -0.426448</td><td style=\"text-align: right;\">                 0.355249</td><td style=\"text-align: right;\">                  0.244394</td><td style=\"text-align: right;\">              -0.215304</td><td style=\"text-align: right;\">          -0.474263 </td><td style=\"text-align: right;\"> -0.0340707 </td><td style=\"text-align: right;\">                 1.58588  </td></tr>\n",
       "<tr><td style=\"text-align: right;\">     5.25537e+06</td><td style=\"text-align: right;\">            0</td><td style=\"text-align: right;\">         1</td><td style=\"text-align: right;\">           1</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">      1</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">         1</td><td style=\"text-align: right;\">             0</td><td style=\"text-align: right;\">     0</td><td style=\"text-align: right;\">       1</td><td style=\"text-align: right;\">       nan</td><td style=\"text-align: right;\">      nan</td><td>False                         </td><td style=\"text-align: right;\">        -0.690283</td><td style=\"text-align: right;\">                -0.182763</td><td style=\"text-align: right;\">                  0.244394</td><td style=\"text-align: right;\">              -0.215304</td><td style=\"text-align: right;\">          -0.597602 </td><td style=\"text-align: right;\"> -0.0247158 </td><td style=\"text-align: right;\">                 0.94601  </td></tr>\n",
       "<tr><td style=\"text-align: right;\">     4.59782e+06</td><td style=\"text-align: right;\">            0</td><td style=\"text-align: right;\">         1</td><td style=\"text-align: right;\">           1</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">      1</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">         0</td><td style=\"text-align: right;\">             1</td><td style=\"text-align: right;\">     1</td><td style=\"text-align: right;\">       0</td><td style=\"text-align: right;\">       nan</td><td style=\"text-align: right;\">      nan</td><td>False                         </td><td style=\"text-align: right;\">         0.716833</td><td style=\"text-align: right;\">                -0.682306</td><td style=\"text-align: right;\">                 -4.09175 </td><td style=\"text-align: right;\">               4.61086 </td><td style=\"text-align: right;\">           0.913302 </td><td style=\"text-align: right;\"> -0.0243693 </td><td style=\"text-align: right;\">                 0.306144 </td></tr>\n",
       "<tr><td style=\"text-align: right;\">     6.13819e+06</td><td style=\"text-align: right;\">            0</td><td style=\"text-align: right;\">         1</td><td style=\"text-align: right;\">           1</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">      1</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">      0</td><td style=\"text-align: right;\">         0</td><td style=\"text-align: right;\">             1</td><td style=\"text-align: right;\">   nan</td><td style=\"text-align: right;\">     nan</td><td style=\"text-align: right;\">         1</td><td style=\"text-align: right;\">        0</td><td>False                         </td><td style=\"text-align: right;\">         0.145192</td><td style=\"text-align: right;\">                -0.246306</td><td style=\"text-align: right;\">                  0.244394</td><td style=\"text-align: right;\">              -0.215304</td><td style=\"text-align: right;\">           0.142433 </td><td style=\"text-align: right;\"> -0.0302594 </td><td style=\"text-align: right;\">                 1.03742  </td></tr>\n",
       "</tbody>\n",
       "</table>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": []
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead>\n",
       "<tr><th>       </th><th>Unnamed: 0        </th><th>high_priced        </th><th>term_360           </th><th>conforming         </th><th>black              </th><th>asian              </th><th>white             </th><th>amind               </th><th>hipac                </th><th>hispanic           </th><th>non_hispanic      </th><th>male              </th><th>female            </th><th>agegte62           </th><th>agelt62           </th><th>debt_to_income_ratio_missing  </th><th>loan_amount_std     </th><th>loan_to_value_ratio_std  </th><th>no_intro_rate_period_std  </th><th>intro_rate_period_std  </th><th>property_value_std  </th><th>income_std           </th><th>debt_to_income_ratio_std  </th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>type   </td><td>int               </td><td>int                </td><td>int                </td><td>int                </td><td>int                </td><td>int                </td><td>int               </td><td>int                 </td><td>int                  </td><td>int                </td><td>int               </td><td>int               </td><td>int               </td><td>int                </td><td>int               </td><td>enum                          </td><td>real                </td><td>real                     </td><td>real                      </td><td>real                   </td><td>real                </td><td>real                 </td><td>real                      </td></tr>\n",
       "<tr><td>mins   </td><td>4056.0            </td><td>0.0                </td><td>0.0                </td><td>0.0                </td><td>0.0                </td><td>0.0                </td><td>0.0               </td><td>0.0                 </td><td>0.0                  </td><td>0.0                </td><td>0.0               </td><td>0.0               </td><td>0.0               </td><td>0.0                </td><td>0.0               </td><td>                              </td><td>-1.26192360553821   </td><td>-3.4586567959595325      </td><td>-4.09174686250922         </td><td>-0.21530422241413488   </td><td>-1.029288791468282  </td><td>-0.05278067773266968 </td><td>-2.527546647809547        </td></tr>\n",
       "<tr><td>mean   </td><td>3323235.642428521 </td><td>0.09752407846301246</td><td>0.9601885936160557 </td><td>0.9262014018455953 </td><td>0.07671265111627497</td><td>0.08197782157249169</td><td>0.8342206665293996</td><td>0.004294496573227049</td><td>0.0027943642086066416</td><td>0.12101442886949366</td><td>0.8789855711305063</td><td>0.6133134578655611</td><td>0.3866865421344389</td><td>0.22066733067729083</td><td>0.7793326693227092</td><td>                              </td><td>0.004096505679623418</td><td>-0.00846253655870321     </td><td>-0.010447651839036788     </td><td>0.0055571501904296584  </td><td>0.005967330951646153</td><td>0.010591915149438086 </td><td>-0.007259301127656858     </td></tr>\n",
       "<tr><td>maxs   </td><td>6504997.0         </td><td>1.0                </td><td>1.0                </td><td>1.0                </td><td>1.0                </td><td>1.0                </td><td>1.0               </td><td>1.0                 </td><td>1.0                  </td><td>1.0                </td><td>1.0               </td><td>1.0               </td><td>1.0               </td><td>1.0                </td><td>1.0               </td><td>                              </td><td>19.05331330602753   </td><td>37.75782848293456        </td><td>0.24439439525512616       </td><td>14.263178448962517     </td><td>27.40037507602021   </td><td>137.15347258538733   </td><td>3.87111034635691          </td></tr>\n",
       "<tr><td>sigma  </td><td>1954622.3871122967</td><td>0.29667381367158185</td><td>0.19551834219733183</td><td>0.26144614833636154</td><td>0.2661388804577039 </td><td>0.27433496324350387</td><td>0.3718879052734942</td><td>0.06539250456663769 </td><td>0.05278861339759911  </td><td>0.3261488353928688 </td><td>0.3261488353928688</td><td>0.487002164716498 </td><td>0.487002164716498 </td><td>0.4147069141947798 </td><td>0.4147069141947798</td><td>                              </td><td>0.9853674098950168  </td><td>0.6456480788175912       </td><td>1.0198592403214601        </td><td>0.9844019248509008     </td><td>0.9922639123326747  </td><td>1.2475615471305674   </td><td>0.9950299052385418        </td></tr>\n",
       "<tr><td>zeros  </td><td>0                 </td><td>35794              </td><td>1579               </td><td>2927               </td><td>31389              </td><td>31210              </td><td>5636              </td><td>33851               </td><td>33902                </td><td>29911              </td><td>4118              </td><td>8301              </td><td>13166             </td><td>15649              </td><td>4431              </td><td>                              </td><td>0                   </td><td>0                        </td><td>0                         </td><td>0                      </td><td>0                   </td><td>0                    </td><td>0                         </td></tr>\n",
       "<tr><td>missing</td><td>0                 </td><td>0                  </td><td>0                  </td><td>0                  </td><td>5665               </td><td>5665               </td><td>5665              </td><td>5665                </td><td>5665                 </td><td>5633               </td><td>5633              </td><td>18195             </td><td>18195             </td><td>19582              </td><td>19582             </td><td>0                             </td><td>0                   </td><td>0                        </td><td>0                         </td><td>0                      </td><td>0                   </td><td>0                    </td><td>0                         </td></tr>\n",
       "<tr><td>0      </td><td>131497.0          </td><td>0.0                </td><td>1.0                </td><td>1.0                </td><td>0.0                </td><td>1.0                </td><td>0.0               </td><td>0.0                 </td><td>0.0                  </td><td>1.0                </td><td>0.0               </td><td>1.0               </td><td>0.0               </td><td>nan                </td><td>nan               </td><td>False                         </td><td>-0.4704208687239607 </td><td>0.3451325282916518       </td><td>0.24439439525512616       </td><td>-0.21530422241413488   </td><td>-0.5050975921761949 </td><td>-0.00808470129089574 </td><td>-2.527546647809547        </td></tr>\n",
       "<tr><td>1      </td><td>851117.0          </td><td>0.0                </td><td>1.0                </td><td>1.0                </td><td>0.0                </td><td>1.0                </td><td>0.0               </td><td>0.0                 </td><td>0.0                  </td><td>0.0                </td><td>1.0               </td><td>1.0               </td><td>0.0               </td><td>nan                </td><td>nan               </td><td>False                         </td><td>0.32108186809028866 </td><td>0.3424391847869839       </td><td>-4.09174686250922         </td><td>3.2032264083275748     </td><td>0.08076315997378489 </td><td>-0.01085654479116079 </td><td>-1.1564058633453065       </td></tr>\n",
       "<tr><td>2      </td><td>835616.0          </td><td>0.0                </td><td>1.0                </td><td>1.0                </td><td>0.0                </td><td>0.0                </td><td>1.0               </td><td>0.0                 </td><td>0.0                  </td><td>0.0                </td><td>1.0               </td><td>nan               </td><td>nan               </td><td>0.0                </td><td>1.0               </td><td>False                         </td><td>-0.11864187458429434</td><td>0.5525199781510916       </td><td>0.24439439525512616       </td><td>-0.21530422241413488   </td><td>-0.2892541571735708 </td><td>-0.01605375135415776 </td><td>1.037419391797479         </td></tr>\n",
       "<tr><td>3      </td><td>121491.0          </td><td>0.0                </td><td>0.0                </td><td>1.0                </td><td>0.0                </td><td>0.0                </td><td>1.0               </td><td>0.0                 </td><td>0.0                  </td><td>0.0                </td><td>1.0               </td><td>1.0               </td><td>0.0               </td><td>1.0                </td><td>0.0               </td><td>False                         </td><td>-0.646310365793794  </td><td>-1.191974453019489       </td><td>0.24439439525512616       </td><td>-0.21530422241413488   </td><td>-0.32008893360251706</td><td>-0.02922000798041675 </td><td>0.03191614985703575       </td></tr>\n",
       "<tr><td>4      </td><td>4479643.0         </td><td>0.0                </td><td>1.0                </td><td>1.0                </td><td>0.0                </td><td>0.0                </td><td>1.0               </td><td>0.0                 </td><td>0.0                  </td><td>0.0                </td><td>1.0               </td><td>nan               </td><td>nan               </td><td>0.0                </td><td>1.0               </td><td>False                         </td><td>0.27710949382283034 </td><td>0.33683703029727435      </td><td>0.24439439525512616       </td><td>-0.21530422241413488   </td><td>0.04992838354483858 </td><td>0.0005773096474325434</td><td>0.7631912349046309        </td></tr>\n",
       "<tr><td>5      </td><td>843194.0          </td><td>1.0                </td><td>1.0                </td><td>1.0                </td><td>0.0                </td><td>1.0                </td><td>0.0               </td><td>0.0                 </td><td>0.0                  </td><td>nan                </td><td>nan               </td><td>nan               </td><td>nan               </td><td>0.0                </td><td>1.0               </td><td>False                         </td><td>0.32108186809028866 </td><td>0.4446785042241832       </td><td>0.24439439525512616       </td><td>-0.21530422241413488   </td><td>0.04992838354483858 </td><td>-0.03511017541847998 </td><td>3.87111034635691          </td></tr>\n",
       "<tr><td>6      </td><td>6377259.0         </td><td>0.0                </td><td>1.0                </td><td>1.0                </td><td>0.0                </td><td>0.0                </td><td>1.0               </td><td>0.0                 </td><td>0.0                  </td><td>0.0                </td><td>1.0               </td><td>0.0               </td><td>1.0               </td><td>nan                </td><td>nan               </td><td>False                         </td><td>-0.42644849445650246</td><td>0.355248989260405        </td><td>0.24439439525512616       </td><td>-0.21530422241413488   </td><td>-0.4742628157472486 </td><td>-0.03407073410588059 </td><td>1.5858757055831754        </td></tr>\n",
       "<tr><td>7      </td><td>5255372.0         </td><td>0.0                </td><td>1.0                </td><td>1.0                </td><td>0.0                </td><td>0.0                </td><td>1.0               </td><td>0.0                 </td><td>0.0                  </td><td>1.0                </td><td>0.0               </td><td>0.0               </td><td>1.0               </td><td>nan                </td><td>nan               </td><td>False                         </td><td>-0.6902827400612522 </td><td>-0.18276279862328496     </td><td>0.24439439525512616       </td><td>-0.21530422241413488   </td><td>-0.5976019214630338 </td><td>-0.02471576229248604 </td><td>0.9460100061665296        </td></tr>\n",
       "<tr><td>8      </td><td>4597825.0         </td><td>0.0                </td><td>1.0                </td><td>1.0                </td><td>0.0                </td><td>1.0                </td><td>0.0               </td><td>0.0                 </td><td>0.0                  </td><td>0.0                </td><td>1.0               </td><td>1.0               </td><td>0.0               </td><td>nan                </td><td>nan               </td><td>False                         </td><td>0.7168332364974134  </td><td>-0.6823060591799226      </td><td>-4.09174686250922         </td><td>4.610856668044749      </td><td>0.913302123555335   </td><td>-0.02436928185495291 </td><td>0.3061443067498839        </td></tr>\n",
       "<tr><td>9      </td><td>6138190.0         </td><td>0.0                </td><td>1.0                </td><td>1.0                </td><td>0.0                </td><td>0.0                </td><td>1.0               </td><td>0.0                 </td><td>0.0                  </td><td>0.0                </td><td>1.0               </td><td>nan               </td><td>nan               </td><td>1.0                </td><td>0.0               </td><td>False                         </td><td>0.14519237102045546 </td><td>-0.24630575464082383     </td><td>0.24439439525512616       </td><td>-0.21530422241413488   </td><td>0.1424327128316775  </td><td>-0.030259449293016143</td><td>1.037419391797479         </td></tr>\n",
       "</tbody>\n",
       "</table>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "test.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setup Predictors, Response, and Fold Column for Modelling Purposes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Inputs and output\n",
    "X = ['term_360',\n",
    "'conforming',\n",
    "'debt_to_income_ratio_missing',\n",
    "'loan_amount_std',\n",
    "'loan_to_value_ratio_std',\n",
    "'no_intro_rate_period_std',\n",
    "'intro_rate_period_std',\n",
    "'property_value_std',\n",
    "'income_std',\n",
    "'debt_to_income_ratio_std']\n",
    "fold_column = 'cv_fold'\n",
    "Y = 'high_priced'\n",
    "\n",
    "# Convert outcome to categorical\n",
    "train[Y] = train[Y].asfactor()\n",
    "test[Y] = test[Y].asfactor()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead>\n",
       "<tr><th style=\"text-align: right;\">  high_priced</th><th style=\"text-align: right;\">  Count</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td style=\"text-align: right;\">            0</td><td style=\"text-align: right;\"> 144833</td></tr>\n",
       "<tr><td style=\"text-align: right;\">            1</td><td style=\"text-align: right;\">  15505</td></tr>\n",
       "</tbody>\n",
       "</table>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": []
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Distribution of outcome in train\n",
    "train[Y].table()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead>\n",
       "<tr><th style=\"text-align: right;\">  high_priced</th><th style=\"text-align: right;\">  Count</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td style=\"text-align: right;\">            0</td><td style=\"text-align: right;\">  35794</td></tr>\n",
       "<tr><td style=\"text-align: right;\">            1</td><td style=\"text-align: right;\">   3868</td></tr>\n",
       "</tbody>\n",
       "</table>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": []
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Distribution of outcome in test\n",
    "test[Y].table()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to get Shapley contributions\n",
    "def get_shapley(model, train, X):\n",
    "    # calculate SHAP values using function predict_contributions\n",
    "    contributions = model.predict_contributions(train[X])\n",
    "    print(\"First 5 rows of contributions:\\n\")\n",
    "    print(contributions.head(5))\n",
    "    # convert the H2O Frame to use with shap's visualization functions\n",
    "    contributions_matrix = contributions.as_data_frame().as_matrix()\n",
    "    # shap values are calculated for all features\n",
    "    shap_values = contributions_matrix[:,0:len(X)]\n",
    "    # expected values is the last returned column\n",
    "    expected_value = contributions_matrix[:,len(X)-1].min()\n",
    "    return shap_values, expected_value"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create Pandas Frame for Correlation Calculation Purposes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_pd_for_corr = train.as_data_frame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>high_priced</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>term_360</th>\n",
       "      <td>0.052448</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>conforming</th>\n",
       "      <td>0.074543</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>debt_to_income_ratio_missing</th>\n",
       "      <td>0.008857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>loan_amount_std</th>\n",
       "      <td>-0.123545</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>loan_to_value_ratio_std</th>\n",
       "      <td>0.111540</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_intro_rate_period_std</th>\n",
       "      <td>0.018108</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>intro_rate_period_std</th>\n",
       "      <td>-0.031716</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>property_value_std</th>\n",
       "      <td>-0.137319</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>income_std</th>\n",
       "      <td>-0.007911</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>debt_to_income_ratio_std</th>\n",
       "      <td>0.128645</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                              high_priced\n",
       "term_360                         0.052448\n",
       "conforming                       0.074543\n",
       "debt_to_income_ratio_missing     0.008857\n",
       "loan_amount_std                 -0.123545\n",
       "loan_to_value_ratio_std          0.111540\n",
       "no_intro_rate_period_std         0.018108\n",
       "intro_rate_period_std           -0.031716\n",
       "property_value_std              -0.137319\n",
       "income_std                      -0.007911\n",
       "debt_to_income_ratio_std         0.128645"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Calculate correlation\n",
    "numeric_corr = pd.DataFrame(train_pd_for_corr[X + [Y]].corr()[Y]).iloc[:-1]\n",
    "numeric_corr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>high_priced</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>loan_amount_std</th>\n",
       "      <td>-0.123545</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>loan_to_value_ratio_std</th>\n",
       "      <td>0.111540</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>property_value_std</th>\n",
       "      <td>-0.137319</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>debt_to_income_ratio_std</th>\n",
       "      <td>0.128645</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          high_priced\n",
       "loan_amount_std             -0.123545\n",
       "loan_to_value_ratio_std      0.111540\n",
       "property_value_std          -0.137319\n",
       "debt_to_income_ratio_std     0.128645"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Filter to correlations > 0.1\n",
    "numeric_corr = numeric_corr[numeric_corr['high_priced'].abs() > 0.1]\n",
    "numeric_corr"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Make monotonicity constraints dictionary, which is based on heuristics known about the dataset:\n",
    "## `debt_to_income_ratio` will be positively constrained to the outcome\n",
    "## `loan_to_value_ratio` will be positively constrained to the outcome\n",
    "## `income` will be negatively constrained to the outcome\n",
    "## `term_360` will be negatively constrained to the outcome"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'debt_to_income_ratio_std': 1,\n",
       " 'loan_to_value_ratio_std': 1,\n",
       " 'income_std': -1,\n",
       " 'term_360': -1}"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mono_constraints = dict()\n",
    "mono_constraints['debt_to_income_ratio_std'] = 1\n",
    "mono_constraints['loan_to_value_ratio_std'] = 1\n",
    "mono_constraints['income_std'] = -1\n",
    "mono_constraints['term_360'] = -1\n",
    "mono_constraints"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Build Default GBM with Monotonic Constraints"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gbm Model Build progress: |███████████████████████████████████████████████| 100%\n"
     ]
    }
   ],
   "source": [
    "mgbm_default = H2OGradientBoostingEstimator(\n",
    "               # monotonicity constraints\n",
    "               monotone_constraints=mono_constraints, \n",
    "               seed=SEED,\n",
    "               categorical_encoding=\"one_hot_explicit\")\n",
    "mgbm_default.train(x=X,\n",
    "              y=Y,\n",
    "              training_frame=train,\n",
    "              fold_column=fold_column)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model Summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model Details\n",
      "=============\n",
      "H2OGradientBoostingEstimator :  Gradient Boosting Machine\n",
      "Model Key:  GBM_model_python_1575925687897_1\n",
      "\n",
      "\n",
      "Model Summary: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>number_of_trees</th>\n",
       "      <th>number_of_internal_trees</th>\n",
       "      <th>model_size_in_bytes</th>\n",
       "      <th>min_depth</th>\n",
       "      <th>max_depth</th>\n",
       "      <th>mean_depth</th>\n",
       "      <th>min_leaves</th>\n",
       "      <th>max_leaves</th>\n",
       "      <th>mean_leaves</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td></td>\n",
       "      <td>50.0</td>\n",
       "      <td>50.0</td>\n",
       "      <td>17381.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>22.98</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     number_of_trees  number_of_internal_trees  model_size_in_bytes  \\\n",
       "0               50.0                      50.0              17381.0   \n",
       "\n",
       "   min_depth  max_depth  mean_depth  min_leaves  max_leaves  mean_leaves  \n",
       "0        5.0        5.0         5.0         9.0        32.0        22.98  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "ModelMetricsBinomial: gbm\n",
      "** Reported on train data. **\n",
      "\n",
      "MSE: 0.07746923148219471\n",
      "RMSE: 0.2783329507661547\n",
      "LogLoss: 0.260311833499752\n",
      "Mean Per-Class Error: 0.2536735673014884\n",
      "AUC: 0.8079388964460537\n",
      "pr_auc: 0.27083899487029695\n",
      "Gini: 0.6158777928921073\n",
      "\n",
      "Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.16050013682564504: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>Error</th>\n",
       "      <th>Rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>113716.0</td>\n",
       "      <td>31117.0</td>\n",
       "      <td>0.2148</td>\n",
       "      <td>(31117.0/144833.0)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>5588.0</td>\n",
       "      <td>9917.0</td>\n",
       "      <td>0.3604</td>\n",
       "      <td>(5588.0/15505.0)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Total</td>\n",
       "      <td>119304.0</td>\n",
       "      <td>41034.0</td>\n",
       "      <td>0.2289</td>\n",
       "      <td>(36705.0/160338.0)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 0        1   Error                 Rate\n",
       "0      0  113716.0  31117.0  0.2148   (31117.0/144833.0)\n",
       "1      1    5588.0   9917.0  0.3604     (5588.0/15505.0)\n",
       "2  Total  119304.0  41034.0  0.2289   (36705.0/160338.0)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Maximum Metrics: Maximum metrics at their respective thresholds\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>metric</th>\n",
       "      <th>threshold</th>\n",
       "      <th>value</th>\n",
       "      <th>idx</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>max f1</td>\n",
       "      <td>0.160500</td>\n",
       "      <td>0.350802</td>\n",
       "      <td>185.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>max f2</td>\n",
       "      <td>0.100428</td>\n",
       "      <td>0.519115</td>\n",
       "      <td>241.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>max f0point5</td>\n",
       "      <td>0.235457</td>\n",
       "      <td>0.297843</td>\n",
       "      <td>117.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>max accuracy</td>\n",
       "      <td>0.629338</td>\n",
       "      <td>0.903529</td>\n",
       "      <td>12.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>max precision</td>\n",
       "      <td>0.755401</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>max recall</td>\n",
       "      <td>0.006322</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>396.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>max specificity</td>\n",
       "      <td>0.755401</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>max absolute_mcc</td>\n",
       "      <td>0.128072</td>\n",
       "      <td>0.298557</td>\n",
       "      <td>215.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>max min_per_class_accuracy</td>\n",
       "      <td>0.138872</td>\n",
       "      <td>0.730628</td>\n",
       "      <td>205.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>max mean_per_class_accuracy</td>\n",
       "      <td>0.091369</td>\n",
       "      <td>0.746326</td>\n",
       "      <td>250.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>max tns</td>\n",
       "      <td>0.755401</td>\n",
       "      <td>144833.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>max fns</td>\n",
       "      <td>0.755401</td>\n",
       "      <td>15504.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>max fps</td>\n",
       "      <td>0.003364</td>\n",
       "      <td>144833.000000</td>\n",
       "      <td>399.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>max tps</td>\n",
       "      <td>0.006322</td>\n",
       "      <td>15505.000000</td>\n",
       "      <td>396.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>max tnr</td>\n",
       "      <td>0.755401</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>max fnr</td>\n",
       "      <td>0.755401</td>\n",
       "      <td>0.999936</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>max fpr</td>\n",
       "      <td>0.003364</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>399.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>max tpr</td>\n",
       "      <td>0.006322</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>396.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         metric  threshold          value    idx\n",
       "0                        max f1   0.160500       0.350802  185.0\n",
       "1                        max f2   0.100428       0.519115  241.0\n",
       "2                  max f0point5   0.235457       0.297843  117.0\n",
       "3                  max accuracy   0.629338       0.903529   12.0\n",
       "4                 max precision   0.755401       1.000000    0.0\n",
       "5                    max recall   0.006322       1.000000  396.0\n",
       "6               max specificity   0.755401       1.000000    0.0\n",
       "7              max absolute_mcc   0.128072       0.298557  215.0\n",
       "8    max min_per_class_accuracy   0.138872       0.730628  205.0\n",
       "9   max mean_per_class_accuracy   0.091369       0.746326  250.0\n",
       "10                      max tns   0.755401  144833.000000    0.0\n",
       "11                      max fns   0.755401   15504.000000    0.0\n",
       "12                      max fps   0.003364  144833.000000  399.0\n",
       "13                      max tps   0.006322   15505.000000  396.0\n",
       "14                      max tnr   0.755401       1.000000    0.0\n",
       "15                      max fnr   0.755401       0.999936    0.0\n",
       "16                      max fpr   0.003364       1.000000  399.0\n",
       "17                      max tpr   0.006322       1.000000  396.0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Gains/Lift Table: Avg response rate:  9.67 %, avg score:  9.69 %\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>group</th>\n",
       "      <th>cumulative_data_fraction</th>\n",
       "      <th>lower_threshold</th>\n",
       "      <th>lift</th>\n",
       "      <th>cumulative_lift</th>\n",
       "      <th>response_rate</th>\n",
       "      <th>score</th>\n",
       "      <th>cumulative_response_rate</th>\n",
       "      <th>cumulative_score</th>\n",
       "      <th>capture_rate</th>\n",
       "      <th>cumulative_capture_rate</th>\n",
       "      <th>gain</th>\n",
       "      <th>cumulative_gain</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "      <td>0.011033</td>\n",
       "      <td>0.366827</td>\n",
       "      <td>4.185524</td>\n",
       "      <td>4.185524</td>\n",
       "      <td>0.404748</td>\n",
       "      <td>0.438297</td>\n",
       "      <td>0.404748</td>\n",
       "      <td>0.438297</td>\n",
       "      <td>0.046179</td>\n",
       "      <td>0.046179</td>\n",
       "      <td>318.552443</td>\n",
       "      <td>318.552443</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td></td>\n",
       "      <td>2</td>\n",
       "      <td>0.021592</td>\n",
       "      <td>0.330887</td>\n",
       "      <td>3.591576</td>\n",
       "      <td>3.895070</td>\n",
       "      <td>0.347312</td>\n",
       "      <td>0.344095</td>\n",
       "      <td>0.376661</td>\n",
       "      <td>0.392230</td>\n",
       "      <td>0.037923</td>\n",
       "      <td>0.084102</td>\n",
       "      <td>259.157599</td>\n",
       "      <td>289.506957</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td></td>\n",
       "      <td>3</td>\n",
       "      <td>0.030716</td>\n",
       "      <td>0.308132</td>\n",
       "      <td>3.209048</td>\n",
       "      <td>3.691283</td>\n",
       "      <td>0.310321</td>\n",
       "      <td>0.321051</td>\n",
       "      <td>0.356954</td>\n",
       "      <td>0.371086</td>\n",
       "      <td>0.029281</td>\n",
       "      <td>0.113383</td>\n",
       "      <td>220.904804</td>\n",
       "      <td>269.128287</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td></td>\n",
       "      <td>4</td>\n",
       "      <td>0.040228</td>\n",
       "      <td>0.293384</td>\n",
       "      <td>3.282012</td>\n",
       "      <td>3.594517</td>\n",
       "      <td>0.317377</td>\n",
       "      <td>0.303693</td>\n",
       "      <td>0.347597</td>\n",
       "      <td>0.355152</td>\n",
       "      <td>0.031216</td>\n",
       "      <td>0.144599</td>\n",
       "      <td>228.201234</td>\n",
       "      <td>259.451736</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td></td>\n",
       "      <td>5</td>\n",
       "      <td>0.050163</td>\n",
       "      <td>0.268439</td>\n",
       "      <td>2.577148</td>\n",
       "      <td>3.393017</td>\n",
       "      <td>0.249215</td>\n",
       "      <td>0.278971</td>\n",
       "      <td>0.328111</td>\n",
       "      <td>0.340064</td>\n",
       "      <td>0.025605</td>\n",
       "      <td>0.170203</td>\n",
       "      <td>157.714837</td>\n",
       "      <td>239.301682</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td></td>\n",
       "      <td>6</td>\n",
       "      <td>0.100681</td>\n",
       "      <td>0.238399</td>\n",
       "      <td>2.700163</td>\n",
       "      <td>3.045367</td>\n",
       "      <td>0.261111</td>\n",
       "      <td>0.252269</td>\n",
       "      <td>0.294493</td>\n",
       "      <td>0.296011</td>\n",
       "      <td>0.136408</td>\n",
       "      <td>0.306611</td>\n",
       "      <td>170.016339</td>\n",
       "      <td>204.536689</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td></td>\n",
       "      <td>7</td>\n",
       "      <td>0.150445</td>\n",
       "      <td>0.214653</td>\n",
       "      <td>2.279723</td>\n",
       "      <td>2.792110</td>\n",
       "      <td>0.220454</td>\n",
       "      <td>0.225940</td>\n",
       "      <td>0.270002</td>\n",
       "      <td>0.272833</td>\n",
       "      <td>0.113447</td>\n",
       "      <td>0.420058</td>\n",
       "      <td>127.972292</td>\n",
       "      <td>179.210957</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td></td>\n",
       "      <td>8</td>\n",
       "      <td>0.200027</td>\n",
       "      <td>0.181454</td>\n",
       "      <td>2.211294</td>\n",
       "      <td>2.648137</td>\n",
       "      <td>0.213836</td>\n",
       "      <td>0.199033</td>\n",
       "      <td>0.256080</td>\n",
       "      <td>0.254540</td>\n",
       "      <td>0.109642</td>\n",
       "      <td>0.529700</td>\n",
       "      <td>121.129398</td>\n",
       "      <td>164.813713</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td></td>\n",
       "      <td>9</td>\n",
       "      <td>0.301276</td>\n",
       "      <td>0.143597</td>\n",
       "      <td>1.816723</td>\n",
       "      <td>2.368727</td>\n",
       "      <td>0.175681</td>\n",
       "      <td>0.163362</td>\n",
       "      <td>0.229061</td>\n",
       "      <td>0.223898</td>\n",
       "      <td>0.183941</td>\n",
       "      <td>0.713641</td>\n",
       "      <td>81.672282</td>\n",
       "      <td>136.872712</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td></td>\n",
       "      <td>10</td>\n",
       "      <td>0.400011</td>\n",
       "      <td>0.096532</td>\n",
       "      <td>1.318842</td>\n",
       "      <td>2.109583</td>\n",
       "      <td>0.127535</td>\n",
       "      <td>0.119017</td>\n",
       "      <td>0.204001</td>\n",
       "      <td>0.198010</td>\n",
       "      <td>0.130216</td>\n",
       "      <td>0.843857</td>\n",
       "      <td>31.884167</td>\n",
       "      <td>110.958284</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td></td>\n",
       "      <td>11</td>\n",
       "      <td>0.500630</td>\n",
       "      <td>0.061079</td>\n",
       "      <td>0.808926</td>\n",
       "      <td>1.848171</td>\n",
       "      <td>0.078225</td>\n",
       "      <td>0.077227</td>\n",
       "      <td>0.178722</td>\n",
       "      <td>0.173735</td>\n",
       "      <td>0.081393</td>\n",
       "      <td>0.925250</td>\n",
       "      <td>-19.107378</td>\n",
       "      <td>84.817144</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td></td>\n",
       "      <td>12</td>\n",
       "      <td>0.600070</td>\n",
       "      <td>0.029790</td>\n",
       "      <td>0.382666</td>\n",
       "      <td>1.605317</td>\n",
       "      <td>0.037005</td>\n",
       "      <td>0.044661</td>\n",
       "      <td>0.155237</td>\n",
       "      <td>0.152345</td>\n",
       "      <td>0.038052</td>\n",
       "      <td>0.963302</td>\n",
       "      <td>-61.733440</td>\n",
       "      <td>60.531671</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td></td>\n",
       "      <td>13</td>\n",
       "      <td>0.700127</td>\n",
       "      <td>0.016801</td>\n",
       "      <td>0.201110</td>\n",
       "      <td>1.404637</td>\n",
       "      <td>0.019448</td>\n",
       "      <td>0.022303</td>\n",
       "      <td>0.135831</td>\n",
       "      <td>0.133761</td>\n",
       "      <td>0.020123</td>\n",
       "      <td>0.983425</td>\n",
       "      <td>-79.888998</td>\n",
       "      <td>40.463713</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td></td>\n",
       "      <td>14</td>\n",
       "      <td>0.800029</td>\n",
       "      <td>0.012205</td>\n",
       "      <td>0.085218</td>\n",
       "      <td>1.239878</td>\n",
       "      <td>0.008241</td>\n",
       "      <td>0.014194</td>\n",
       "      <td>0.119899</td>\n",
       "      <td>0.118830</td>\n",
       "      <td>0.008513</td>\n",
       "      <td>0.991938</td>\n",
       "      <td>-91.478220</td>\n",
       "      <td>23.987814</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td></td>\n",
       "      <td>15</td>\n",
       "      <td>0.900491</td>\n",
       "      <td>0.009430</td>\n",
       "      <td>0.052643</td>\n",
       "      <td>1.107425</td>\n",
       "      <td>0.005091</td>\n",
       "      <td>0.010840</td>\n",
       "      <td>0.107090</td>\n",
       "      <td>0.106782</td>\n",
       "      <td>0.005289</td>\n",
       "      <td>0.997227</td>\n",
       "      <td>-94.735745</td>\n",
       "      <td>10.742494</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td></td>\n",
       "      <td>16</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.002667</td>\n",
       "      <td>0.027870</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.002695</td>\n",
       "      <td>0.007902</td>\n",
       "      <td>0.096702</td>\n",
       "      <td>0.096943</td>\n",
       "      <td>0.002773</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-97.213004</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      group  cumulative_data_fraction  lower_threshold      lift  \\\n",
       "0         1                  0.011033         0.366827  4.185524   \n",
       "1         2                  0.021592         0.330887  3.591576   \n",
       "2         3                  0.030716         0.308132  3.209048   \n",
       "3         4                  0.040228         0.293384  3.282012   \n",
       "4         5                  0.050163         0.268439  2.577148   \n",
       "5         6                  0.100681         0.238399  2.700163   \n",
       "6         7                  0.150445         0.214653  2.279723   \n",
       "7         8                  0.200027         0.181454  2.211294   \n",
       "8         9                  0.301276         0.143597  1.816723   \n",
       "9        10                  0.400011         0.096532  1.318842   \n",
       "10       11                  0.500630         0.061079  0.808926   \n",
       "11       12                  0.600070         0.029790  0.382666   \n",
       "12       13                  0.700127         0.016801  0.201110   \n",
       "13       14                  0.800029         0.012205  0.085218   \n",
       "14       15                  0.900491         0.009430  0.052643   \n",
       "15       16                  1.000000         0.002667  0.027870   \n",
       "\n",
       "    cumulative_lift  response_rate     score  cumulative_response_rate  \\\n",
       "0          4.185524       0.404748  0.438297                  0.404748   \n",
       "1          3.895070       0.347312  0.344095                  0.376661   \n",
       "2          3.691283       0.310321  0.321051                  0.356954   \n",
       "3          3.594517       0.317377  0.303693                  0.347597   \n",
       "4          3.393017       0.249215  0.278971                  0.328111   \n",
       "5          3.045367       0.261111  0.252269                  0.294493   \n",
       "6          2.792110       0.220454  0.225940                  0.270002   \n",
       "7          2.648137       0.213836  0.199033                  0.256080   \n",
       "8          2.368727       0.175681  0.163362                  0.229061   \n",
       "9          2.109583       0.127535  0.119017                  0.204001   \n",
       "10         1.848171       0.078225  0.077227                  0.178722   \n",
       "11         1.605317       0.037005  0.044661                  0.155237   \n",
       "12         1.404637       0.019448  0.022303                  0.135831   \n",
       "13         1.239878       0.008241  0.014194                  0.119899   \n",
       "14         1.107425       0.005091  0.010840                  0.107090   \n",
       "15         1.000000       0.002695  0.007902                  0.096702   \n",
       "\n",
       "    cumulative_score  capture_rate  cumulative_capture_rate        gain  \\\n",
       "0           0.438297      0.046179                 0.046179  318.552443   \n",
       "1           0.392230      0.037923                 0.084102  259.157599   \n",
       "2           0.371086      0.029281                 0.113383  220.904804   \n",
       "3           0.355152      0.031216                 0.144599  228.201234   \n",
       "4           0.340064      0.025605                 0.170203  157.714837   \n",
       "5           0.296011      0.136408                 0.306611  170.016339   \n",
       "6           0.272833      0.113447                 0.420058  127.972292   \n",
       "7           0.254540      0.109642                 0.529700  121.129398   \n",
       "8           0.223898      0.183941                 0.713641   81.672282   \n",
       "9           0.198010      0.130216                 0.843857   31.884167   \n",
       "10          0.173735      0.081393                 0.925250  -19.107378   \n",
       "11          0.152345      0.038052                 0.963302  -61.733440   \n",
       "12          0.133761      0.020123                 0.983425  -79.888998   \n",
       "13          0.118830      0.008513                 0.991938  -91.478220   \n",
       "14          0.106782      0.005289                 0.997227  -94.735745   \n",
       "15          0.096943      0.002773                 1.000000  -97.213004   \n",
       "\n",
       "    cumulative_gain  \n",
       "0        318.552443  \n",
       "1        289.506957  \n",
       "2        269.128287  \n",
       "3        259.451736  \n",
       "4        239.301682  \n",
       "5        204.536689  \n",
       "6        179.210957  \n",
       "7        164.813713  \n",
       "8        136.872712  \n",
       "9        110.958284  \n",
       "10        84.817144  \n",
       "11        60.531671  \n",
       "12        40.463713  \n",
       "13        23.987814  \n",
       "14        10.742494  \n",
       "15         0.000000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "ModelMetricsBinomial: gbm\n",
      "** Reported on cross-validation data. **\n",
      "\n",
      "MSE: 0.07786691732767274\n",
      "RMSE: 0.2790464429582874\n",
      "LogLoss: 0.26184112441054336\n",
      "Mean Per-Class Error: 0.2551517057866107\n",
      "AUC: 0.8047385674202855\n",
      "pr_auc: 0.26197778676858546\n",
      "Gini: 0.609477134840571\n",
      "\n",
      "Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.16144778019404407: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>Error</th>\n",
       "      <th>Rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>113586.0</td>\n",
       "      <td>31247.0</td>\n",
       "      <td>0.2157</td>\n",
       "      <td>(31247.0/144833.0)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>5678.0</td>\n",
       "      <td>9827.0</td>\n",
       "      <td>0.3662</td>\n",
       "      <td>(5678.0/15505.0)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Total</td>\n",
       "      <td>119264.0</td>\n",
       "      <td>41074.0</td>\n",
       "      <td>0.2303</td>\n",
       "      <td>(36925.0/160338.0)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 0        1   Error                 Rate\n",
       "0      0  113586.0  31247.0  0.2157   (31247.0/144833.0)\n",
       "1      1    5678.0   9827.0  0.3662     (5678.0/15505.0)\n",
       "2  Total  119264.0  41074.0  0.2303   (36925.0/160338.0)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Maximum Metrics: Maximum metrics at their respective thresholds\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>metric</th>\n",
       "      <th>threshold</th>\n",
       "      <th>value</th>\n",
       "      <th>idx</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>max f1</td>\n",
       "      <td>0.161448</td>\n",
       "      <td>0.347373</td>\n",
       "      <td>203.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>max f2</td>\n",
       "      <td>0.087112</td>\n",
       "      <td>0.516665</td>\n",
       "      <td>279.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>max f0point5</td>\n",
       "      <td>0.232498</td>\n",
       "      <td>0.293769</td>\n",
       "      <td>139.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>max accuracy</td>\n",
       "      <td>0.611152</td>\n",
       "      <td>0.903348</td>\n",
       "      <td>14.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>max precision</td>\n",
       "      <td>0.862930</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>max recall</td>\n",
       "      <td>0.005448</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>399.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>max specificity</td>\n",
       "      <td>0.862930</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>max absolute_mcc</td>\n",
       "      <td>0.122144</td>\n",
       "      <td>0.294907</td>\n",
       "      <td>242.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>max min_per_class_accuracy</td>\n",
       "      <td>0.136550</td>\n",
       "      <td>0.729700</td>\n",
       "      <td>228.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>max mean_per_class_accuracy</td>\n",
       "      <td>0.087112</td>\n",
       "      <td>0.744848</td>\n",
       "      <td>279.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>max tns</td>\n",
       "      <td>0.862930</td>\n",
       "      <td>144833.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>max fns</td>\n",
       "      <td>0.862930</td>\n",
       "      <td>15504.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>max fps</td>\n",
       "      <td>0.005448</td>\n",
       "      <td>144833.000000</td>\n",
       "      <td>399.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>max tps</td>\n",
       "      <td>0.005448</td>\n",
       "      <td>15505.000000</td>\n",
       "      <td>399.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>max tnr</td>\n",
       "      <td>0.862930</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>max fnr</td>\n",
       "      <td>0.862930</td>\n",
       "      <td>0.999936</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>max fpr</td>\n",
       "      <td>0.005448</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>399.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>max tpr</td>\n",
       "      <td>0.005448</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>399.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         metric  threshold          value    idx\n",
       "0                        max f1   0.161448       0.347373  203.0\n",
       "1                        max f2   0.087112       0.516665  279.0\n",
       "2                  max f0point5   0.232498       0.293769  139.0\n",
       "3                  max accuracy   0.611152       0.903348   14.0\n",
       "4                 max precision   0.862930       1.000000    0.0\n",
       "5                    max recall   0.005448       1.000000  399.0\n",
       "6               max specificity   0.862930       1.000000    0.0\n",
       "7              max absolute_mcc   0.122144       0.294907  242.0\n",
       "8    max min_per_class_accuracy   0.136550       0.729700  228.0\n",
       "9   max mean_per_class_accuracy   0.087112       0.744848  279.0\n",
       "10                      max tns   0.862930  144833.000000    0.0\n",
       "11                      max fns   0.862930   15504.000000    0.0\n",
       "12                      max fps   0.005448  144833.000000  399.0\n",
       "13                      max tps   0.005448   15505.000000  399.0\n",
       "14                      max tnr   0.862930       1.000000    0.0\n",
       "15                      max fnr   0.862930       0.999936    0.0\n",
       "16                      max fpr   0.005448       1.000000  399.0\n",
       "17                      max tpr   0.005448       1.000000  399.0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Gains/Lift Table: Avg response rate:  9.67 %, avg score:  9.69 %\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>group</th>\n",
       "      <th>cumulative_data_fraction</th>\n",
       "      <th>lower_threshold</th>\n",
       "      <th>lift</th>\n",
       "      <th>cumulative_lift</th>\n",
       "      <th>response_rate</th>\n",
       "      <th>score</th>\n",
       "      <th>cumulative_response_rate</th>\n",
       "      <th>cumulative_score</th>\n",
       "      <th>capture_rate</th>\n",
       "      <th>cumulative_capture_rate</th>\n",
       "      <th>gain</th>\n",
       "      <th>cumulative_gain</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "      <td>0.010166</td>\n",
       "      <td>0.375019</td>\n",
       "      <td>3.895341</td>\n",
       "      <td>3.895341</td>\n",
       "      <td>0.376687</td>\n",
       "      <td>0.453095</td>\n",
       "      <td>0.376687</td>\n",
       "      <td>0.453095</td>\n",
       "      <td>0.039600</td>\n",
       "      <td>0.039600</td>\n",
       "      <td>289.534079</td>\n",
       "      <td>289.534079</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td></td>\n",
       "      <td>2</td>\n",
       "      <td>0.020008</td>\n",
       "      <td>0.332917</td>\n",
       "      <td>3.669828</td>\n",
       "      <td>3.784412</td>\n",
       "      <td>0.354880</td>\n",
       "      <td>0.351174</td>\n",
       "      <td>0.365960</td>\n",
       "      <td>0.402960</td>\n",
       "      <td>0.036117</td>\n",
       "      <td>0.075718</td>\n",
       "      <td>266.982808</td>\n",
       "      <td>278.441216</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td></td>\n",
       "      <td>3</td>\n",
       "      <td>0.030149</td>\n",
       "      <td>0.312497</td>\n",
       "      <td>3.090868</td>\n",
       "      <td>3.551126</td>\n",
       "      <td>0.298893</td>\n",
       "      <td>0.324982</td>\n",
       "      <td>0.343401</td>\n",
       "      <td>0.376731</td>\n",
       "      <td>0.031345</td>\n",
       "      <td>0.107062</td>\n",
       "      <td>209.086772</td>\n",
       "      <td>255.112642</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td></td>\n",
       "      <td>4</td>\n",
       "      <td>0.040065</td>\n",
       "      <td>0.291343</td>\n",
       "      <td>3.167353</td>\n",
       "      <td>3.456139</td>\n",
       "      <td>0.306289</td>\n",
       "      <td>0.303840</td>\n",
       "      <td>0.334215</td>\n",
       "      <td>0.358690</td>\n",
       "      <td>0.031409</td>\n",
       "      <td>0.138471</td>\n",
       "      <td>216.735344</td>\n",
       "      <td>245.613902</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td></td>\n",
       "      <td>5</td>\n",
       "      <td>0.050269</td>\n",
       "      <td>0.270604</td>\n",
       "      <td>2.787533</td>\n",
       "      <td>3.320427</td>\n",
       "      <td>0.269560</td>\n",
       "      <td>0.279100</td>\n",
       "      <td>0.321092</td>\n",
       "      <td>0.342535</td>\n",
       "      <td>0.028442</td>\n",
       "      <td>0.166914</td>\n",
       "      <td>178.753277</td>\n",
       "      <td>232.042689</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td></td>\n",
       "      <td>6</td>\n",
       "      <td>0.100001</td>\n",
       "      <td>0.238947</td>\n",
       "      <td>2.646863</td>\n",
       "      <td>2.985451</td>\n",
       "      <td>0.255957</td>\n",
       "      <td>0.254313</td>\n",
       "      <td>0.288699</td>\n",
       "      <td>0.298661</td>\n",
       "      <td>0.131635</td>\n",
       "      <td>0.298549</td>\n",
       "      <td>164.686301</td>\n",
       "      <td>198.545131</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td></td>\n",
       "      <td>7</td>\n",
       "      <td>0.150070</td>\n",
       "      <td>0.212385</td>\n",
       "      <td>2.361129</td>\n",
       "      <td>2.777154</td>\n",
       "      <td>0.228326</td>\n",
       "      <td>0.225365</td>\n",
       "      <td>0.268556</td>\n",
       "      <td>0.274206</td>\n",
       "      <td>0.118220</td>\n",
       "      <td>0.416769</td>\n",
       "      <td>136.112942</td>\n",
       "      <td>177.715374</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td></td>\n",
       "      <td>8</td>\n",
       "      <td>0.200034</td>\n",
       "      <td>0.183508</td>\n",
       "      <td>2.146694</td>\n",
       "      <td>2.619681</td>\n",
       "      <td>0.207590</td>\n",
       "      <td>0.197471</td>\n",
       "      <td>0.253328</td>\n",
       "      <td>0.255040</td>\n",
       "      <td>0.107256</td>\n",
       "      <td>0.524025</td>\n",
       "      <td>114.669433</td>\n",
       "      <td>161.968140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td></td>\n",
       "      <td>9</td>\n",
       "      <td>0.300197</td>\n",
       "      <td>0.143312</td>\n",
       "      <td>1.824816</td>\n",
       "      <td>2.354467</td>\n",
       "      <td>0.176463</td>\n",
       "      <td>0.163053</td>\n",
       "      <td>0.227682</td>\n",
       "      <td>0.224347</td>\n",
       "      <td>0.182780</td>\n",
       "      <td>0.706804</td>\n",
       "      <td>82.481565</td>\n",
       "      <td>135.446743</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td></td>\n",
       "      <td>10</td>\n",
       "      <td>0.400192</td>\n",
       "      <td>0.097110</td>\n",
       "      <td>1.333185</td>\n",
       "      <td>2.099282</td>\n",
       "      <td>0.128922</td>\n",
       "      <td>0.119111</td>\n",
       "      <td>0.203005</td>\n",
       "      <td>0.198052</td>\n",
       "      <td>0.133312</td>\n",
       "      <td>0.840116</td>\n",
       "      <td>33.318487</td>\n",
       "      <td>109.928208</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td></td>\n",
       "      <td>11</td>\n",
       "      <td>0.500006</td>\n",
       "      <td>0.060236</td>\n",
       "      <td>0.819323</td>\n",
       "      <td>1.843769</td>\n",
       "      <td>0.079230</td>\n",
       "      <td>0.076985</td>\n",
       "      <td>0.178296</td>\n",
       "      <td>0.173884</td>\n",
       "      <td>0.081780</td>\n",
       "      <td>0.921896</td>\n",
       "      <td>-18.067652</td>\n",
       "      <td>84.376933</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td></td>\n",
       "      <td>12</td>\n",
       "      <td>0.600001</td>\n",
       "      <td>0.029997</td>\n",
       "      <td>0.387636</td>\n",
       "      <td>1.601093</td>\n",
       "      <td>0.037485</td>\n",
       "      <td>0.043994</td>\n",
       "      <td>0.154829</td>\n",
       "      <td>0.152237</td>\n",
       "      <td>0.038762</td>\n",
       "      <td>0.960658</td>\n",
       "      <td>-61.236376</td>\n",
       "      <td>60.109309</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td></td>\n",
       "      <td>13</td>\n",
       "      <td>0.699996</td>\n",
       "      <td>0.016802</td>\n",
       "      <td>0.209620</td>\n",
       "      <td>1.402320</td>\n",
       "      <td>0.020271</td>\n",
       "      <td>0.022396</td>\n",
       "      <td>0.135607</td>\n",
       "      <td>0.133689</td>\n",
       "      <td>0.020961</td>\n",
       "      <td>0.981619</td>\n",
       "      <td>-79.037974</td>\n",
       "      <td>40.232011</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td></td>\n",
       "      <td>14</td>\n",
       "      <td>0.800060</td>\n",
       "      <td>0.012262</td>\n",
       "      <td>0.088302</td>\n",
       "      <td>1.237976</td>\n",
       "      <td>0.008539</td>\n",
       "      <td>0.014291</td>\n",
       "      <td>0.119715</td>\n",
       "      <td>0.118756</td>\n",
       "      <td>0.008836</td>\n",
       "      <td>0.990455</td>\n",
       "      <td>-91.169758</td>\n",
       "      <td>23.797571</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td></td>\n",
       "      <td>15</td>\n",
       "      <td>0.900017</td>\n",
       "      <td>0.009526</td>\n",
       "      <td>0.061297</td>\n",
       "      <td>1.107292</td>\n",
       "      <td>0.005927</td>\n",
       "      <td>0.010923</td>\n",
       "      <td>0.107077</td>\n",
       "      <td>0.106780</td>\n",
       "      <td>0.006127</td>\n",
       "      <td>0.996582</td>\n",
       "      <td>-93.870345</td>\n",
       "      <td>10.729157</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td></td>\n",
       "      <td>16</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.002775</td>\n",
       "      <td>0.034188</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.003306</td>\n",
       "      <td>0.007796</td>\n",
       "      <td>0.096702</td>\n",
       "      <td>0.096883</td>\n",
       "      <td>0.003418</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-96.581151</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      group  cumulative_data_fraction  lower_threshold      lift  \\\n",
       "0         1                  0.010166         0.375019  3.895341   \n",
       "1         2                  0.020008         0.332917  3.669828   \n",
       "2         3                  0.030149         0.312497  3.090868   \n",
       "3         4                  0.040065         0.291343  3.167353   \n",
       "4         5                  0.050269         0.270604  2.787533   \n",
       "5         6                  0.100001         0.238947  2.646863   \n",
       "6         7                  0.150070         0.212385  2.361129   \n",
       "7         8                  0.200034         0.183508  2.146694   \n",
       "8         9                  0.300197         0.143312  1.824816   \n",
       "9        10                  0.400192         0.097110  1.333185   \n",
       "10       11                  0.500006         0.060236  0.819323   \n",
       "11       12                  0.600001         0.029997  0.387636   \n",
       "12       13                  0.699996         0.016802  0.209620   \n",
       "13       14                  0.800060         0.012262  0.088302   \n",
       "14       15                  0.900017         0.009526  0.061297   \n",
       "15       16                  1.000000         0.002775  0.034188   \n",
       "\n",
       "    cumulative_lift  response_rate     score  cumulative_response_rate  \\\n",
       "0          3.895341       0.376687  0.453095                  0.376687   \n",
       "1          3.784412       0.354880  0.351174                  0.365960   \n",
       "2          3.551126       0.298893  0.324982                  0.343401   \n",
       "3          3.456139       0.306289  0.303840                  0.334215   \n",
       "4          3.320427       0.269560  0.279100                  0.321092   \n",
       "5          2.985451       0.255957  0.254313                  0.288699   \n",
       "6          2.777154       0.228326  0.225365                  0.268556   \n",
       "7          2.619681       0.207590  0.197471                  0.253328   \n",
       "8          2.354467       0.176463  0.163053                  0.227682   \n",
       "9          2.099282       0.128922  0.119111                  0.203005   \n",
       "10         1.843769       0.079230  0.076985                  0.178296   \n",
       "11         1.601093       0.037485  0.043994                  0.154829   \n",
       "12         1.402320       0.020271  0.022396                  0.135607   \n",
       "13         1.237976       0.008539  0.014291                  0.119715   \n",
       "14         1.107292       0.005927  0.010923                  0.107077   \n",
       "15         1.000000       0.003306  0.007796                  0.096702   \n",
       "\n",
       "    cumulative_score  capture_rate  cumulative_capture_rate        gain  \\\n",
       "0           0.453095      0.039600                 0.039600  289.534079   \n",
       "1           0.402960      0.036117                 0.075718  266.982808   \n",
       "2           0.376731      0.031345                 0.107062  209.086772   \n",
       "3           0.358690      0.031409                 0.138471  216.735344   \n",
       "4           0.342535      0.028442                 0.166914  178.753277   \n",
       "5           0.298661      0.131635                 0.298549  164.686301   \n",
       "6           0.274206      0.118220                 0.416769  136.112942   \n",
       "7           0.255040      0.107256                 0.524025  114.669433   \n",
       "8           0.224347      0.182780                 0.706804   82.481565   \n",
       "9           0.198052      0.133312                 0.840116   33.318487   \n",
       "10          0.173884      0.081780                 0.921896  -18.067652   \n",
       "11          0.152237      0.038762                 0.960658  -61.236376   \n",
       "12          0.133689      0.020961                 0.981619  -79.037974   \n",
       "13          0.118756      0.008836                 0.990455  -91.169758   \n",
       "14          0.106780      0.006127                 0.996582  -93.870345   \n",
       "15          0.096883      0.003418                 1.000000  -96.581151   \n",
       "\n",
       "    cumulative_gain  \n",
       "0        289.534079  \n",
       "1        278.441216  \n",
       "2        255.112642  \n",
       "3        245.613902  \n",
       "4        232.042689  \n",
       "5        198.545131  \n",
       "6        177.715374  \n",
       "7        161.968140  \n",
       "8        135.446743  \n",
       "9        109.928208  \n",
       "10        84.376933  \n",
       "11        60.109309  \n",
       "12        40.232011  \n",
       "13        23.797571  \n",
       "14        10.729157  \n",
       "15         0.000000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Cross-Validation Metrics Summary: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>sd</th>\n",
       "      <th>cv_1_valid</th>\n",
       "      <th>cv_2_valid</th>\n",
       "      <th>cv_3_valid</th>\n",
       "      <th>cv_4_valid</th>\n",
       "      <th>cv_5_valid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>accuracy</td>\n",
       "      <td>0.7704801</td>\n",
       "      <td>0.014431337</td>\n",
       "      <td>0.7920445</td>\n",
       "      <td>0.75499684</td>\n",
       "      <td>0.76842666</td>\n",
       "      <td>0.7608514</td>\n",
       "      <td>0.77608097</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>auc</td>\n",
       "      <td>0.8048742</td>\n",
       "      <td>0.00280107</td>\n",
       "      <td>0.80185527</td>\n",
       "      <td>0.80827004</td>\n",
       "      <td>0.803526</td>\n",
       "      <td>0.8074152</td>\n",
       "      <td>0.80330443</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>err</td>\n",
       "      <td>0.22951993</td>\n",
       "      <td>0.014431337</td>\n",
       "      <td>0.20795546</td>\n",
       "      <td>0.24500315</td>\n",
       "      <td>0.23157336</td>\n",
       "      <td>0.23914862</td>\n",
       "      <td>0.22391905</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>err_count</td>\n",
       "      <td>7360.0</td>\n",
       "      <td>460.4655</td>\n",
       "      <td>6650.0</td>\n",
       "      <td>7796.0</td>\n",
       "      <td>7421.0</td>\n",
       "      <td>7719.0</td>\n",
       "      <td>7214.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>f0point5</td>\n",
       "      <td>0.27453804</td>\n",
       "      <td>0.007236497</td>\n",
       "      <td>0.28003556</td>\n",
       "      <td>0.2797435</td>\n",
       "      <td>0.26934677</td>\n",
       "      <td>0.26436567</td>\n",
       "      <td>0.2791987</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>f1</td>\n",
       "      <td>0.34848258</td>\n",
       "      <td>0.00741789</td>\n",
       "      <td>0.34662998</td>\n",
       "      <td>0.35972404</td>\n",
       "      <td>0.34414494</td>\n",
       "      <td>0.34053823</td>\n",
       "      <td>0.35137564</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>f2</td>\n",
       "      <td>0.4774484</td>\n",
       "      <td>0.017465457</td>\n",
       "      <td>0.45477983</td>\n",
       "      <td>0.5037494</td>\n",
       "      <td>0.4764585</td>\n",
       "      <td>0.4783736</td>\n",
       "      <td>0.47388077</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>lift_top_group</td>\n",
       "      <td>3.920367</td>\n",
       "      <td>0.25636455</td>\n",
       "      <td>3.5213163</td>\n",
       "      <td>3.8844738</td>\n",
       "      <td>4.192557</td>\n",
       "      <td>4.088432</td>\n",
       "      <td>3.9150565</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>logloss</td>\n",
       "      <td>0.26185217</td>\n",
       "      <td>0.004189886</td>\n",
       "      <td>0.26224622</td>\n",
       "      <td>0.2665671</td>\n",
       "      <td>0.2588397</td>\n",
       "      <td>0.2565283</td>\n",
       "      <td>0.26507956</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>max_per_class_error</td>\n",
       "      <td>0.36508095</td>\n",
       "      <td>0.042322766</td>\n",
       "      <td>0.42578125</td>\n",
       "      <td>0.3128334</td>\n",
       "      <td>0.35932872</td>\n",
       "      <td>0.34483892</td>\n",
       "      <td>0.38262242</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>mcc</td>\n",
       "      <td>0.28474918</td>\n",
       "      <td>0.008952044</td>\n",
       "      <td>0.27602404</td>\n",
       "      <td>0.29987624</td>\n",
       "      <td>0.28281388</td>\n",
       "      <td>0.2815465</td>\n",
       "      <td>0.28348517</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>mean_per_class_accuracy</td>\n",
       "      <td>0.7099367</td>\n",
       "      <td>0.011065928</td>\n",
       "      <td>0.69470644</td>\n",
       "      <td>0.7248567</td>\n",
       "      <td>0.7112413</td>\n",
       "      <td>0.71350497</td>\n",
       "      <td>0.70537406</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>mean_per_class_error</td>\n",
       "      <td>0.29006332</td>\n",
       "      <td>0.011065928</td>\n",
       "      <td>0.3052936</td>\n",
       "      <td>0.27514333</td>\n",
       "      <td>0.28875872</td>\n",
       "      <td>0.28649506</td>\n",
       "      <td>0.29462597</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>mse</td>\n",
       "      <td>0.07787084</td>\n",
       "      <td>0.0014590838</td>\n",
       "      <td>0.077674255</td>\n",
       "      <td>0.07984063</td>\n",
       "      <td>0.07670607</td>\n",
       "      <td>0.076330654</td>\n",
       "      <td>0.07880258</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>precision</td>\n",
       "      <td>0.24055189</td>\n",
       "      <td>0.007614233</td>\n",
       "      <td>0.24824092</td>\n",
       "      <td>0.24363111</td>\n",
       "      <td>0.23525858</td>\n",
       "      <td>0.23005886</td>\n",
       "      <td>0.24556994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>r2</td>\n",
       "      <td>0.10846628</td>\n",
       "      <td>0.0037383859</td>\n",
       "      <td>0.10552032</td>\n",
       "      <td>0.11411903</td>\n",
       "      <td>0.1063986</td>\n",
       "      <td>0.10582397</td>\n",
       "      <td>0.11046948</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>recall</td>\n",
       "      <td>0.63491905</td>\n",
       "      <td>0.042322766</td>\n",
       "      <td>0.57421875</td>\n",
       "      <td>0.68716663</td>\n",
       "      <td>0.64067125</td>\n",
       "      <td>0.6551611</td>\n",
       "      <td>0.6173776</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>rmse</td>\n",
       "      <td>0.27904367</td>\n",
       "      <td>0.0026113384</td>\n",
       "      <td>0.278701</td>\n",
       "      <td>0.28256083</td>\n",
       "      <td>0.27695858</td>\n",
       "      <td>0.27628005</td>\n",
       "      <td>0.28071797</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>specificity</td>\n",
       "      <td>0.78495425</td>\n",
       "      <td>0.020426754</td>\n",
       "      <td>0.8151941</td>\n",
       "      <td>0.7625467</td>\n",
       "      <td>0.7818113</td>\n",
       "      <td>0.7718488</td>\n",
       "      <td>0.7933705</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   mean            sd   cv_1_valid  \\\n",
       "0                  accuracy   0.7704801   0.014431337    0.7920445   \n",
       "1                       auc   0.8048742    0.00280107   0.80185527   \n",
       "2                       err  0.22951993   0.014431337   0.20795546   \n",
       "3                 err_count      7360.0      460.4655       6650.0   \n",
       "4                  f0point5  0.27453804   0.007236497   0.28003556   \n",
       "5                        f1  0.34848258    0.00741789   0.34662998   \n",
       "6                        f2   0.4774484   0.017465457   0.45477983   \n",
       "7            lift_top_group    3.920367    0.25636455    3.5213163   \n",
       "8                   logloss  0.26185217   0.004189886   0.26224622   \n",
       "9       max_per_class_error  0.36508095   0.042322766   0.42578125   \n",
       "10                      mcc  0.28474918   0.008952044   0.27602404   \n",
       "11  mean_per_class_accuracy   0.7099367   0.011065928   0.69470644   \n",
       "12     mean_per_class_error  0.29006332   0.011065928    0.3052936   \n",
       "13                      mse  0.07787084  0.0014590838  0.077674255   \n",
       "14                precision  0.24055189   0.007614233   0.24824092   \n",
       "15                       r2  0.10846628  0.0037383859   0.10552032   \n",
       "16                   recall  0.63491905   0.042322766   0.57421875   \n",
       "17                     rmse  0.27904367  0.0026113384     0.278701   \n",
       "18              specificity  0.78495425   0.020426754    0.8151941   \n",
       "\n",
       "    cv_2_valid  cv_3_valid   cv_4_valid  cv_5_valid  \n",
       "0   0.75499684  0.76842666    0.7608514  0.77608097  \n",
       "1   0.80827004    0.803526    0.8074152  0.80330443  \n",
       "2   0.24500315  0.23157336   0.23914862  0.22391905  \n",
       "3       7796.0      7421.0       7719.0      7214.0  \n",
       "4    0.2797435  0.26934677   0.26436567   0.2791987  \n",
       "5   0.35972404  0.34414494   0.34053823  0.35137564  \n",
       "6    0.5037494   0.4764585    0.4783736  0.47388077  \n",
       "7    3.8844738    4.192557     4.088432   3.9150565  \n",
       "8    0.2665671   0.2588397    0.2565283  0.26507956  \n",
       "9    0.3128334  0.35932872   0.34483892  0.38262242  \n",
       "10  0.29987624  0.28281388    0.2815465  0.28348517  \n",
       "11   0.7248567   0.7112413   0.71350497  0.70537406  \n",
       "12  0.27514333  0.28875872   0.28649506  0.29462597  \n",
       "13  0.07984063  0.07670607  0.076330654  0.07880258  \n",
       "14  0.24363111  0.23525858   0.23005886  0.24556994  \n",
       "15  0.11411903   0.1063986   0.10582397  0.11046948  \n",
       "16  0.68716663  0.64067125    0.6551611   0.6173776  \n",
       "17  0.28256083  0.27695858   0.27628005  0.28071797  \n",
       "18   0.7625467   0.7818113    0.7718488   0.7933705  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Scoring History: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>timestamp</th>\n",
       "      <th>duration</th>\n",
       "      <th>number_of_trees</th>\n",
       "      <th>training_rmse</th>\n",
       "      <th>training_logloss</th>\n",
       "      <th>training_auc</th>\n",
       "      <th>training_pr_auc</th>\n",
       "      <th>training_lift</th>\n",
       "      <th>training_classification_error</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:20:21</td>\n",
       "      <td>15.676 sec</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.295552</td>\n",
       "      <td>0.317775</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.903298</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:20:21</td>\n",
       "      <td>16.054 sec</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.293001</td>\n",
       "      <td>0.309425</td>\n",
       "      <td>0.774476</td>\n",
       "      <td>0.227195</td>\n",
       "      <td>3.199757</td>\n",
       "      <td>0.254943</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:20:22</td>\n",
       "      <td>16.358 sec</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.291030</td>\n",
       "      <td>0.303305</td>\n",
       "      <td>0.775249</td>\n",
       "      <td>0.239186</td>\n",
       "      <td>3.942622</td>\n",
       "      <td>0.246118</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:20:22</td>\n",
       "      <td>16.397 sec</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.289515</td>\n",
       "      <td>0.298726</td>\n",
       "      <td>0.776227</td>\n",
       "      <td>0.239274</td>\n",
       "      <td>3.500892</td>\n",
       "      <td>0.226210</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:20:22</td>\n",
       "      <td>16.423 sec</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.288318</td>\n",
       "      <td>0.295140</td>\n",
       "      <td>0.774524</td>\n",
       "      <td>0.239349</td>\n",
       "      <td>4.016215</td>\n",
       "      <td>0.226235</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:20:22</td>\n",
       "      <td>16.449 sec</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.287294</td>\n",
       "      <td>0.292073</td>\n",
       "      <td>0.775406</td>\n",
       "      <td>0.242985</td>\n",
       "      <td>4.064141</td>\n",
       "      <td>0.228187</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:20:22</td>\n",
       "      <td>16.478 sec</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.286586</td>\n",
       "      <td>0.289781</td>\n",
       "      <td>0.776410</td>\n",
       "      <td>0.243305</td>\n",
       "      <td>4.061651</td>\n",
       "      <td>0.220671</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:20:22</td>\n",
       "      <td>16.506 sec</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.285554</td>\n",
       "      <td>0.286630</td>\n",
       "      <td>0.780791</td>\n",
       "      <td>0.248294</td>\n",
       "      <td>4.061651</td>\n",
       "      <td>0.227894</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:20:22</td>\n",
       "      <td>16.535 sec</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.284760</td>\n",
       "      <td>0.283998</td>\n",
       "      <td>0.784599</td>\n",
       "      <td>0.249709</td>\n",
       "      <td>4.044661</td>\n",
       "      <td>0.235434</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:20:22</td>\n",
       "      <td>16.572 sec</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.284024</td>\n",
       "      <td>0.281708</td>\n",
       "      <td>0.786704</td>\n",
       "      <td>0.251037</td>\n",
       "      <td>4.042575</td>\n",
       "      <td>0.231636</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:20:22</td>\n",
       "      <td>16.649 sec</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.283407</td>\n",
       "      <td>0.279750</td>\n",
       "      <td>0.788016</td>\n",
       "      <td>0.251605</td>\n",
       "      <td>3.969036</td>\n",
       "      <td>0.221476</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:20:22</td>\n",
       "      <td>16.697 sec</td>\n",
       "      <td>11.0</td>\n",
       "      <td>0.282926</td>\n",
       "      <td>0.278187</td>\n",
       "      <td>0.789097</td>\n",
       "      <td>0.252325</td>\n",
       "      <td>3.969036</td>\n",
       "      <td>0.222143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:20:22</td>\n",
       "      <td>16.744 sec</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.282549</td>\n",
       "      <td>0.276844</td>\n",
       "      <td>0.789978</td>\n",
       "      <td>0.252489</td>\n",
       "      <td>3.969036</td>\n",
       "      <td>0.222181</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:20:22</td>\n",
       "      <td>16.787 sec</td>\n",
       "      <td>13.0</td>\n",
       "      <td>0.282189</td>\n",
       "      <td>0.275607</td>\n",
       "      <td>0.790571</td>\n",
       "      <td>0.252970</td>\n",
       "      <td>3.969036</td>\n",
       "      <td>0.222274</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:20:22</td>\n",
       "      <td>16.833 sec</td>\n",
       "      <td>14.0</td>\n",
       "      <td>0.282043</td>\n",
       "      <td>0.275040</td>\n",
       "      <td>0.790746</td>\n",
       "      <td>0.252980</td>\n",
       "      <td>3.965127</td>\n",
       "      <td>0.223285</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:20:22</td>\n",
       "      <td>16.885 sec</td>\n",
       "      <td>15.0</td>\n",
       "      <td>0.281767</td>\n",
       "      <td>0.274069</td>\n",
       "      <td>0.791440</td>\n",
       "      <td>0.253410</td>\n",
       "      <td>3.979777</td>\n",
       "      <td>0.223185</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:20:22</td>\n",
       "      <td>16.936 sec</td>\n",
       "      <td>16.0</td>\n",
       "      <td>0.281571</td>\n",
       "      <td>0.273407</td>\n",
       "      <td>0.791894</td>\n",
       "      <td>0.253887</td>\n",
       "      <td>3.984684</td>\n",
       "      <td>0.228168</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:20:22</td>\n",
       "      <td>16.987 sec</td>\n",
       "      <td>17.0</td>\n",
       "      <td>0.281178</td>\n",
       "      <td>0.271559</td>\n",
       "      <td>0.794658</td>\n",
       "      <td>0.254642</td>\n",
       "      <td>3.972941</td>\n",
       "      <td>0.228523</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:20:22</td>\n",
       "      <td>17.040 sec</td>\n",
       "      <td>18.0</td>\n",
       "      <td>0.280841</td>\n",
       "      <td>0.270160</td>\n",
       "      <td>0.796740</td>\n",
       "      <td>0.255690</td>\n",
       "      <td>3.970508</td>\n",
       "      <td>0.228923</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:20:22</td>\n",
       "      <td>17.092 sec</td>\n",
       "      <td>19.0</td>\n",
       "      <td>0.280537</td>\n",
       "      <td>0.269045</td>\n",
       "      <td>0.798519</td>\n",
       "      <td>0.256800</td>\n",
       "      <td>3.976350</td>\n",
       "      <td>0.229079</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                timestamp    duration  number_of_trees  training_rmse  \\\n",
       "0     2019-12-09 13:20:21  15.676 sec              0.0       0.295552   \n",
       "1     2019-12-09 13:20:21  16.054 sec              1.0       0.293001   \n",
       "2     2019-12-09 13:20:22  16.358 sec              2.0       0.291030   \n",
       "3     2019-12-09 13:20:22  16.397 sec              3.0       0.289515   \n",
       "4     2019-12-09 13:20:22  16.423 sec              4.0       0.288318   \n",
       "5     2019-12-09 13:20:22  16.449 sec              5.0       0.287294   \n",
       "6     2019-12-09 13:20:22  16.478 sec              6.0       0.286586   \n",
       "7     2019-12-09 13:20:22  16.506 sec              7.0       0.285554   \n",
       "8     2019-12-09 13:20:22  16.535 sec              8.0       0.284760   \n",
       "9     2019-12-09 13:20:22  16.572 sec              9.0       0.284024   \n",
       "10    2019-12-09 13:20:22  16.649 sec             10.0       0.283407   \n",
       "11    2019-12-09 13:20:22  16.697 sec             11.0       0.282926   \n",
       "12    2019-12-09 13:20:22  16.744 sec             12.0       0.282549   \n",
       "13    2019-12-09 13:20:22  16.787 sec             13.0       0.282189   \n",
       "14    2019-12-09 13:20:22  16.833 sec             14.0       0.282043   \n",
       "15    2019-12-09 13:20:22  16.885 sec             15.0       0.281767   \n",
       "16    2019-12-09 13:20:22  16.936 sec             16.0       0.281571   \n",
       "17    2019-12-09 13:20:22  16.987 sec             17.0       0.281178   \n",
       "18    2019-12-09 13:20:22  17.040 sec             18.0       0.280841   \n",
       "19    2019-12-09 13:20:22  17.092 sec             19.0       0.280537   \n",
       "\n",
       "    training_logloss  training_auc  training_pr_auc  training_lift  \\\n",
       "0           0.317775      0.500000         0.000000       1.000000   \n",
       "1           0.309425      0.774476         0.227195       3.199757   \n",
       "2           0.303305      0.775249         0.239186       3.942622   \n",
       "3           0.298726      0.776227         0.239274       3.500892   \n",
       "4           0.295140      0.774524         0.239349       4.016215   \n",
       "5           0.292073      0.775406         0.242985       4.064141   \n",
       "6           0.289781      0.776410         0.243305       4.061651   \n",
       "7           0.286630      0.780791         0.248294       4.061651   \n",
       "8           0.283998      0.784599         0.249709       4.044661   \n",
       "9           0.281708      0.786704         0.251037       4.042575   \n",
       "10          0.279750      0.788016         0.251605       3.969036   \n",
       "11          0.278187      0.789097         0.252325       3.969036   \n",
       "12          0.276844      0.789978         0.252489       3.969036   \n",
       "13          0.275607      0.790571         0.252970       3.969036   \n",
       "14          0.275040      0.790746         0.252980       3.965127   \n",
       "15          0.274069      0.791440         0.253410       3.979777   \n",
       "16          0.273407      0.791894         0.253887       3.984684   \n",
       "17          0.271559      0.794658         0.254642       3.972941   \n",
       "18          0.270160      0.796740         0.255690       3.970508   \n",
       "19          0.269045      0.798519         0.256800       3.976350   \n",
       "\n",
       "    training_classification_error  \n",
       "0                        0.903298  \n",
       "1                        0.254943  \n",
       "2                        0.246118  \n",
       "3                        0.226210  \n",
       "4                        0.226235  \n",
       "5                        0.228187  \n",
       "6                        0.220671  \n",
       "7                        0.227894  \n",
       "8                        0.235434  \n",
       "9                        0.231636  \n",
       "10                       0.221476  \n",
       "11                       0.222143  \n",
       "12                       0.222181  \n",
       "13                       0.222274  \n",
       "14                       0.223285  \n",
       "15                       0.223185  \n",
       "16                       0.228168  \n",
       "17                       0.228523  \n",
       "18                       0.228923  \n",
       "19                       0.229079  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "See the whole table with table.as_data_frame()\n",
      "\n",
      "Variable Importances: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>variable</th>\n",
       "      <th>relative_importance</th>\n",
       "      <th>scaled_importance</th>\n",
       "      <th>percentage</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>loan_to_value_ratio_std</td>\n",
       "      <td>3650.476562</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>4.375255e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>property_value_std</td>\n",
       "      <td>2176.975830</td>\n",
       "      <td>5.963539e-01</td>\n",
       "      <td>2.609200e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>debt_to_income_ratio_std</td>\n",
       "      <td>1193.050049</td>\n",
       "      <td>3.268204e-01</td>\n",
       "      <td>1.429922e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>intro_rate_period_std</td>\n",
       "      <td>602.766052</td>\n",
       "      <td>1.651198e-01</td>\n",
       "      <td>7.224413e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>loan_amount_std</td>\n",
       "      <td>525.585449</td>\n",
       "      <td>1.439772e-01</td>\n",
       "      <td>6.299370e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>no_intro_rate_period_std</td>\n",
       "      <td>149.259399</td>\n",
       "      <td>4.088765e-02</td>\n",
       "      <td>1.788939e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>conforming</td>\n",
       "      <td>33.390831</td>\n",
       "      <td>9.146979e-03</td>\n",
       "      <td>4.002036e-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>income_std</td>\n",
       "      <td>7.539486</td>\n",
       "      <td>2.065343e-03</td>\n",
       "      <td>9.036401e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>debt_to_income_ratio_missing.False</td>\n",
       "      <td>4.415467</td>\n",
       "      <td>1.209559e-03</td>\n",
       "      <td>5.292129e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>term_360</td>\n",
       "      <td>0.001692</td>\n",
       "      <td>4.635131e-07</td>\n",
       "      <td>2.027988e-07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>debt_to_income_ratio_missing.True</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>debt_to_income_ratio_missing.missing(NA)</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    variable  relative_importance  \\\n",
       "0                    loan_to_value_ratio_std          3650.476562   \n",
       "1                         property_value_std          2176.975830   \n",
       "2                   debt_to_income_ratio_std          1193.050049   \n",
       "3                      intro_rate_period_std           602.766052   \n",
       "4                            loan_amount_std           525.585449   \n",
       "5                   no_intro_rate_period_std           149.259399   \n",
       "6                                 conforming            33.390831   \n",
       "7                                 income_std             7.539486   \n",
       "8         debt_to_income_ratio_missing.False             4.415467   \n",
       "9                                   term_360             0.001692   \n",
       "10         debt_to_income_ratio_missing.True             0.000000   \n",
       "11  debt_to_income_ratio_missing.missing(NA)             0.000000   \n",
       "\n",
       "    scaled_importance    percentage  \n",
       "0        1.000000e+00  4.375255e-01  \n",
       "1        5.963539e-01  2.609200e-01  \n",
       "2        3.268204e-01  1.429922e-01  \n",
       "3        1.651198e-01  7.224413e-02  \n",
       "4        1.439772e-01  6.299370e-02  \n",
       "5        4.088765e-02  1.788939e-02  \n",
       "6        9.146979e-03  4.002036e-03  \n",
       "7        2.065343e-03  9.036401e-04  \n",
       "8        1.209559e-03  5.292129e-04  \n",
       "9        4.635131e-07  2.027988e-07  \n",
       "10       0.000000e+00  0.000000e+00  \n",
       "11       0.000000e+00  0.000000e+00  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": []
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mgbm_default"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.808905130264245\n"
     ]
    }
   ],
   "source": [
    "# Get the AUC on the test set for MGBM\n",
    "perf_mgbm_default = mgbm_default.model_performance(test)\n",
    "print(perf_mgbm_default.auc())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8047385674202855\n"
     ]
    }
   ],
   "source": [
    "# Get the AUC on 5 fold CV for MGBM\n",
    "perf_cv_mgbm_default = mgbm_default.model_performance(xval=True)\n",
    "print(perf_cv_mgbm_default.auc())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Build Default GBM with NO Monotonic Constraints"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gbm Model Build progress: |███████████████████████████████████████████████| 100%\n"
     ]
    }
   ],
   "source": [
    "gbm_default = H2OGradientBoostingEstimator(\n",
    "               seed=SEED,\n",
    "               categorical_encoding=\"one_hot_explicit\")\n",
    "gbm_default.train(x=X,\n",
    "              y=Y,\n",
    "              training_frame=train,\n",
    "              fold_column=fold_column)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model Summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model Details\n",
      "=============\n",
      "H2OGradientBoostingEstimator :  Gradient Boosting Machine\n",
      "Model Key:  GBM_model_python_1575925687897_418\n",
      "\n",
      "\n",
      "Model Summary: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>number_of_trees</th>\n",
       "      <th>number_of_internal_trees</th>\n",
       "      <th>model_size_in_bytes</th>\n",
       "      <th>min_depth</th>\n",
       "      <th>max_depth</th>\n",
       "      <th>mean_depth</th>\n",
       "      <th>min_leaves</th>\n",
       "      <th>max_leaves</th>\n",
       "      <th>mean_leaves</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td></td>\n",
       "      <td>50.0</td>\n",
       "      <td>50.0</td>\n",
       "      <td>21720.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>29.9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     number_of_trees  number_of_internal_trees  model_size_in_bytes  \\\n",
       "0               50.0                      50.0              21720.0   \n",
       "\n",
       "   min_depth  max_depth  mean_depth  min_leaves  max_leaves  mean_leaves  \n",
       "0        5.0        5.0         5.0        22.0        32.0         29.9  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "ModelMetricsBinomial: gbm\n",
      "** Reported on train data. **\n",
      "\n",
      "MSE: 0.0757316619042703\n",
      "RMSE: 0.27519386240298005\n",
      "LogLoss: 0.2526668544940987\n",
      "Mean Per-Class Error: 0.23803431310394685\n",
      "AUC: 0.8241574690166849\n",
      "pr_auc: 0.30286940310761046\n",
      "Gini: 0.6483149380333697\n",
      "\n",
      "Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.1748981761202983: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>Error</th>\n",
       "      <th>Rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>113444.0</td>\n",
       "      <td>31389.0</td>\n",
       "      <td>0.2167</td>\n",
       "      <td>(31389.0/144833.0)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>4936.0</td>\n",
       "      <td>10569.0</td>\n",
       "      <td>0.3183</td>\n",
       "      <td>(4936.0/15505.0)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Total</td>\n",
       "      <td>118380.0</td>\n",
       "      <td>41958.0</td>\n",
       "      <td>0.2266</td>\n",
       "      <td>(36325.0/160338.0)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 0        1   Error                 Rate\n",
       "0      0  113444.0  31389.0  0.2167   (31389.0/144833.0)\n",
       "1      1    4936.0  10569.0  0.3183     (4936.0/15505.0)\n",
       "2  Total  118380.0  41958.0  0.2266   (36325.0/160338.0)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Maximum Metrics: Maximum metrics at their respective thresholds\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>metric</th>\n",
       "      <th>threshold</th>\n",
       "      <th>value</th>\n",
       "      <th>idx</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>max f1</td>\n",
       "      <td>0.174898</td>\n",
       "      <td>0.367854</td>\n",
       "      <td>200.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>max f2</td>\n",
       "      <td>0.106860</td>\n",
       "      <td>0.538358</td>\n",
       "      <td>261.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>max f0point5</td>\n",
       "      <td>0.257875</td>\n",
       "      <td>0.320982</td>\n",
       "      <td>120.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>max accuracy</td>\n",
       "      <td>0.411694</td>\n",
       "      <td>0.904402</td>\n",
       "      <td>44.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>max precision</td>\n",
       "      <td>0.796284</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>max recall</td>\n",
       "      <td>0.005666</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>398.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>max specificity</td>\n",
       "      <td>0.796284</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>max absolute_mcc</td>\n",
       "      <td>0.133277</td>\n",
       "      <td>0.321932</td>\n",
       "      <td>240.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>max min_per_class_accuracy</td>\n",
       "      <td>0.155944</td>\n",
       "      <td>0.747482</td>\n",
       "      <td>219.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>max mean_per_class_accuracy</td>\n",
       "      <td>0.106860</td>\n",
       "      <td>0.761966</td>\n",
       "      <td>261.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>max tns</td>\n",
       "      <td>0.796284</td>\n",
       "      <td>144833.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>max fns</td>\n",
       "      <td>0.796284</td>\n",
       "      <td>15504.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>max fps</td>\n",
       "      <td>0.005130</td>\n",
       "      <td>144833.000000</td>\n",
       "      <td>399.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>max tps</td>\n",
       "      <td>0.005666</td>\n",
       "      <td>15505.000000</td>\n",
       "      <td>398.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>max tnr</td>\n",
       "      <td>0.796284</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>max fnr</td>\n",
       "      <td>0.796284</td>\n",
       "      <td>0.999936</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>max fpr</td>\n",
       "      <td>0.005130</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>399.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>max tpr</td>\n",
       "      <td>0.005666</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>398.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         metric  threshold          value    idx\n",
       "0                        max f1   0.174898       0.367854  200.0\n",
       "1                        max f2   0.106860       0.538358  261.0\n",
       "2                  max f0point5   0.257875       0.320982  120.0\n",
       "3                  max accuracy   0.411694       0.904402   44.0\n",
       "4                 max precision   0.796284       1.000000    0.0\n",
       "5                    max recall   0.005666       1.000000  398.0\n",
       "6               max specificity   0.796284       1.000000    0.0\n",
       "7              max absolute_mcc   0.133277       0.321932  240.0\n",
       "8    max min_per_class_accuracy   0.155944       0.747482  219.0\n",
       "9   max mean_per_class_accuracy   0.106860       0.761966  261.0\n",
       "10                      max tns   0.796284  144833.000000    0.0\n",
       "11                      max fns   0.796284   15504.000000    0.0\n",
       "12                      max fps   0.005130  144833.000000  399.0\n",
       "13                      max tps   0.005666   15505.000000  398.0\n",
       "14                      max tnr   0.796284       1.000000    0.0\n",
       "15                      max fnr   0.796284       0.999936    0.0\n",
       "16                      max fpr   0.005130       1.000000  399.0\n",
       "17                      max tpr   0.005666       1.000000  398.0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Gains/Lift Table: Avg response rate:  9.67 %, avg score:  9.68 %\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>group</th>\n",
       "      <th>cumulative_data_fraction</th>\n",
       "      <th>lower_threshold</th>\n",
       "      <th>lift</th>\n",
       "      <th>cumulative_lift</th>\n",
       "      <th>response_rate</th>\n",
       "      <th>score</th>\n",
       "      <th>cumulative_response_rate</th>\n",
       "      <th>cumulative_score</th>\n",
       "      <th>capture_rate</th>\n",
       "      <th>cumulative_capture_rate</th>\n",
       "      <th>gain</th>\n",
       "      <th>cumulative_gain</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "      <td>0.010291</td>\n",
       "      <td>0.366205</td>\n",
       "      <td>5.201862</td>\n",
       "      <td>5.201862</td>\n",
       "      <td>0.503030</td>\n",
       "      <td>0.423953</td>\n",
       "      <td>0.503030</td>\n",
       "      <td>0.423953</td>\n",
       "      <td>0.053531</td>\n",
       "      <td>0.053531</td>\n",
       "      <td>420.186216</td>\n",
       "      <td>420.186216</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td></td>\n",
       "      <td>2</td>\n",
       "      <td>0.020089</td>\n",
       "      <td>0.337407</td>\n",
       "      <td>3.686180</td>\n",
       "      <td>4.462608</td>\n",
       "      <td>0.356461</td>\n",
       "      <td>0.349156</td>\n",
       "      <td>0.431543</td>\n",
       "      <td>0.387471</td>\n",
       "      <td>0.036117</td>\n",
       "      <td>0.089649</td>\n",
       "      <td>268.617996</td>\n",
       "      <td>346.260828</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td></td>\n",
       "      <td>3</td>\n",
       "      <td>0.030323</td>\n",
       "      <td>0.311893</td>\n",
       "      <td>3.459620</td>\n",
       "      <td>4.124084</td>\n",
       "      <td>0.334552</td>\n",
       "      <td>0.323703</td>\n",
       "      <td>0.398807</td>\n",
       "      <td>0.365949</td>\n",
       "      <td>0.035408</td>\n",
       "      <td>0.125056</td>\n",
       "      <td>245.962044</td>\n",
       "      <td>312.408441</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td></td>\n",
       "      <td>4</td>\n",
       "      <td>0.040059</td>\n",
       "      <td>0.291681</td>\n",
       "      <td>3.332190</td>\n",
       "      <td>3.931628</td>\n",
       "      <td>0.322229</td>\n",
       "      <td>0.301962</td>\n",
       "      <td>0.380196</td>\n",
       "      <td>0.350398</td>\n",
       "      <td>0.032441</td>\n",
       "      <td>0.157498</td>\n",
       "      <td>233.219013</td>\n",
       "      <td>293.162809</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td></td>\n",
       "      <td>5</td>\n",
       "      <td>0.050007</td>\n",
       "      <td>0.280969</td>\n",
       "      <td>3.170391</td>\n",
       "      <td>3.780197</td>\n",
       "      <td>0.306583</td>\n",
       "      <td>0.285037</td>\n",
       "      <td>0.365553</td>\n",
       "      <td>0.337396</td>\n",
       "      <td>0.031538</td>\n",
       "      <td>0.189036</td>\n",
       "      <td>217.039127</td>\n",
       "      <td>278.019722</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td></td>\n",
       "      <td>6</td>\n",
       "      <td>0.100020</td>\n",
       "      <td>0.252807</td>\n",
       "      <td>2.773862</td>\n",
       "      <td>3.276998</td>\n",
       "      <td>0.268238</td>\n",
       "      <td>0.264928</td>\n",
       "      <td>0.316892</td>\n",
       "      <td>0.301159</td>\n",
       "      <td>0.138729</td>\n",
       "      <td>0.327765</td>\n",
       "      <td>177.386224</td>\n",
       "      <td>227.699835</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td></td>\n",
       "      <td>7</td>\n",
       "      <td>0.150002</td>\n",
       "      <td>0.223107</td>\n",
       "      <td>2.405256</td>\n",
       "      <td>2.986526</td>\n",
       "      <td>0.232593</td>\n",
       "      <td>0.238116</td>\n",
       "      <td>0.288803</td>\n",
       "      <td>0.280153</td>\n",
       "      <td>0.120219</td>\n",
       "      <td>0.447985</td>\n",
       "      <td>140.525575</td>\n",
       "      <td>198.652622</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td></td>\n",
       "      <td>8</td>\n",
       "      <td>0.200327</td>\n",
       "      <td>0.199909</td>\n",
       "      <td>2.256859</td>\n",
       "      <td>2.803223</td>\n",
       "      <td>0.218243</td>\n",
       "      <td>0.210968</td>\n",
       "      <td>0.271077</td>\n",
       "      <td>0.262773</td>\n",
       "      <td>0.113576</td>\n",
       "      <td>0.561561</td>\n",
       "      <td>125.685851</td>\n",
       "      <td>180.322333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td></td>\n",
       "      <td>9</td>\n",
       "      <td>0.300328</td>\n",
       "      <td>0.155734</td>\n",
       "      <td>1.870986</td>\n",
       "      <td>2.492813</td>\n",
       "      <td>0.180928</td>\n",
       "      <td>0.178507</td>\n",
       "      <td>0.241060</td>\n",
       "      <td>0.234714</td>\n",
       "      <td>0.187101</td>\n",
       "      <td>0.748662</td>\n",
       "      <td>87.098601</td>\n",
       "      <td>149.281312</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td></td>\n",
       "      <td>10</td>\n",
       "      <td>0.399999</td>\n",
       "      <td>0.084520</td>\n",
       "      <td>1.228813</td>\n",
       "      <td>2.177853</td>\n",
       "      <td>0.118829</td>\n",
       "      <td>0.122466</td>\n",
       "      <td>0.210603</td>\n",
       "      <td>0.206745</td>\n",
       "      <td>0.122477</td>\n",
       "      <td>0.871138</td>\n",
       "      <td>22.881274</td>\n",
       "      <td>117.785265</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td></td>\n",
       "      <td>11</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.045277</td>\n",
       "      <td>0.673323</td>\n",
       "      <td>1.876943</td>\n",
       "      <td>0.065112</td>\n",
       "      <td>0.065484</td>\n",
       "      <td>0.181504</td>\n",
       "      <td>0.178492</td>\n",
       "      <td>0.067333</td>\n",
       "      <td>0.938471</td>\n",
       "      <td>-32.667722</td>\n",
       "      <td>87.694292</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td></td>\n",
       "      <td>12</td>\n",
       "      <td>0.600082</td>\n",
       "      <td>0.023612</td>\n",
       "      <td>0.316412</td>\n",
       "      <td>1.616676</td>\n",
       "      <td>0.030598</td>\n",
       "      <td>0.032225</td>\n",
       "      <td>0.156336</td>\n",
       "      <td>0.154097</td>\n",
       "      <td>0.031667</td>\n",
       "      <td>0.970139</td>\n",
       "      <td>-68.358845</td>\n",
       "      <td>61.667595</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td></td>\n",
       "      <td>13</td>\n",
       "      <td>0.700595</td>\n",
       "      <td>0.012899</td>\n",
       "      <td>0.148224</td>\n",
       "      <td>1.406001</td>\n",
       "      <td>0.014334</td>\n",
       "      <td>0.017650</td>\n",
       "      <td>0.135963</td>\n",
       "      <td>0.134522</td>\n",
       "      <td>0.014898</td>\n",
       "      <td>0.985037</td>\n",
       "      <td>-85.177570</td>\n",
       "      <td>40.600075</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td></td>\n",
       "      <td>14</td>\n",
       "      <td>0.800946</td>\n",
       "      <td>0.009364</td>\n",
       "      <td>0.075839</td>\n",
       "      <td>1.239345</td>\n",
       "      <td>0.007334</td>\n",
       "      <td>0.010785</td>\n",
       "      <td>0.119847</td>\n",
       "      <td>0.119019</td>\n",
       "      <td>0.007610</td>\n",
       "      <td>0.992648</td>\n",
       "      <td>-92.416134</td>\n",
       "      <td>23.934466</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td></td>\n",
       "      <td>15</td>\n",
       "      <td>0.900074</td>\n",
       "      <td>0.007182</td>\n",
       "      <td>0.046194</td>\n",
       "      <td>1.107939</td>\n",
       "      <td>0.004467</td>\n",
       "      <td>0.008189</td>\n",
       "      <td>0.107140</td>\n",
       "      <td>0.106813</td>\n",
       "      <td>0.004579</td>\n",
       "      <td>0.997227</td>\n",
       "      <td>-95.380555</td>\n",
       "      <td>10.793907</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td></td>\n",
       "      <td>16</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.003965</td>\n",
       "      <td>0.027753</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.002684</td>\n",
       "      <td>0.006419</td>\n",
       "      <td>0.096702</td>\n",
       "      <td>0.096781</td>\n",
       "      <td>0.002773</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-97.224659</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      group  cumulative_data_fraction  lower_threshold      lift  \\\n",
       "0         1                  0.010291         0.366205  5.201862   \n",
       "1         2                  0.020089         0.337407  3.686180   \n",
       "2         3                  0.030323         0.311893  3.459620   \n",
       "3         4                  0.040059         0.291681  3.332190   \n",
       "4         5                  0.050007         0.280969  3.170391   \n",
       "5         6                  0.100020         0.252807  2.773862   \n",
       "6         7                  0.150002         0.223107  2.405256   \n",
       "7         8                  0.200327         0.199909  2.256859   \n",
       "8         9                  0.300328         0.155734  1.870986   \n",
       "9        10                  0.399999         0.084520  1.228813   \n",
       "10       11                  0.500000         0.045277  0.673323   \n",
       "11       12                  0.600082         0.023612  0.316412   \n",
       "12       13                  0.700595         0.012899  0.148224   \n",
       "13       14                  0.800946         0.009364  0.075839   \n",
       "14       15                  0.900074         0.007182  0.046194   \n",
       "15       16                  1.000000         0.003965  0.027753   \n",
       "\n",
       "    cumulative_lift  response_rate     score  cumulative_response_rate  \\\n",
       "0          5.201862       0.503030  0.423953                  0.503030   \n",
       "1          4.462608       0.356461  0.349156                  0.431543   \n",
       "2          4.124084       0.334552  0.323703                  0.398807   \n",
       "3          3.931628       0.322229  0.301962                  0.380196   \n",
       "4          3.780197       0.306583  0.285037                  0.365553   \n",
       "5          3.276998       0.268238  0.264928                  0.316892   \n",
       "6          2.986526       0.232593  0.238116                  0.288803   \n",
       "7          2.803223       0.218243  0.210968                  0.271077   \n",
       "8          2.492813       0.180928  0.178507                  0.241060   \n",
       "9          2.177853       0.118829  0.122466                  0.210603   \n",
       "10         1.876943       0.065112  0.065484                  0.181504   \n",
       "11         1.616676       0.030598  0.032225                  0.156336   \n",
       "12         1.406001       0.014334  0.017650                  0.135963   \n",
       "13         1.239345       0.007334  0.010785                  0.119847   \n",
       "14         1.107939       0.004467  0.008189                  0.107140   \n",
       "15         1.000000       0.002684  0.006419                  0.096702   \n",
       "\n",
       "    cumulative_score  capture_rate  cumulative_capture_rate        gain  \\\n",
       "0           0.423953      0.053531                 0.053531  420.186216   \n",
       "1           0.387471      0.036117                 0.089649  268.617996   \n",
       "2           0.365949      0.035408                 0.125056  245.962044   \n",
       "3           0.350398      0.032441                 0.157498  233.219013   \n",
       "4           0.337396      0.031538                 0.189036  217.039127   \n",
       "5           0.301159      0.138729                 0.327765  177.386224   \n",
       "6           0.280153      0.120219                 0.447985  140.525575   \n",
       "7           0.262773      0.113576                 0.561561  125.685851   \n",
       "8           0.234714      0.187101                 0.748662   87.098601   \n",
       "9           0.206745      0.122477                 0.871138   22.881274   \n",
       "10          0.178492      0.067333                 0.938471  -32.667722   \n",
       "11          0.154097      0.031667                 0.970139  -68.358845   \n",
       "12          0.134522      0.014898                 0.985037  -85.177570   \n",
       "13          0.119019      0.007610                 0.992648  -92.416134   \n",
       "14          0.106813      0.004579                 0.997227  -95.380555   \n",
       "15          0.096781      0.002773                 1.000000  -97.224659   \n",
       "\n",
       "    cumulative_gain  \n",
       "0        420.186216  \n",
       "1        346.260828  \n",
       "2        312.408441  \n",
       "3        293.162809  \n",
       "4        278.019722  \n",
       "5        227.699835  \n",
       "6        198.652622  \n",
       "7        180.322333  \n",
       "8        149.281312  \n",
       "9        117.785265  \n",
       "10        87.694292  \n",
       "11        61.667595  \n",
       "12        40.600075  \n",
       "13        23.934466  \n",
       "14        10.793907  \n",
       "15         0.000000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "ModelMetricsBinomial: gbm\n",
      "** Reported on cross-validation data. **\n",
      "\n",
      "MSE: 0.0763104038609322\n",
      "RMSE: 0.27624337794946724\n",
      "LogLoss: 0.254824328692756\n",
      "Mean Per-Class Error: 0.2411510506536242\n",
      "AUC: 0.8198539766244762\n",
      "pr_auc: 0.2862189867678687\n",
      "Gini: 0.6397079532489525\n",
      "\n",
      "Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.1728701422882463: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>Error</th>\n",
       "      <th>Rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>112706.0</td>\n",
       "      <td>32127.0</td>\n",
       "      <td>0.2218</td>\n",
       "      <td>(32127.0/144833.0)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>4919.0</td>\n",
       "      <td>10586.0</td>\n",
       "      <td>0.3173</td>\n",
       "      <td>(4919.0/15505.0)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Total</td>\n",
       "      <td>117625.0</td>\n",
       "      <td>42713.0</td>\n",
       "      <td>0.231</td>\n",
       "      <td>(37046.0/160338.0)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 0        1   Error                 Rate\n",
       "0      0  112706.0  32127.0  0.2218   (32127.0/144833.0)\n",
       "1      1    4919.0  10586.0  0.3173     (4919.0/15505.0)\n",
       "2  Total  117625.0  42713.0   0.231   (37046.0/160338.0)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Maximum Metrics: Maximum metrics at their respective thresholds\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>metric</th>\n",
       "      <th>threshold</th>\n",
       "      <th>value</th>\n",
       "      <th>idx</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>max f1</td>\n",
       "      <td>0.172870</td>\n",
       "      <td>0.363668</td>\n",
       "      <td>212.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>max f2</td>\n",
       "      <td>0.108866</td>\n",
       "      <td>0.534668</td>\n",
       "      <td>269.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>max f0point5</td>\n",
       "      <td>0.257389</td>\n",
       "      <td>0.310773</td>\n",
       "      <td>130.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>max accuracy</td>\n",
       "      <td>0.451089</td>\n",
       "      <td>0.903622</td>\n",
       "      <td>38.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>max precision</td>\n",
       "      <td>0.869474</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>max recall</td>\n",
       "      <td>0.005437</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>399.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>max specificity</td>\n",
       "      <td>0.869474</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>max absolute_mcc</td>\n",
       "      <td>0.126929</td>\n",
       "      <td>0.318723</td>\n",
       "      <td>252.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>max min_per_class_accuracy</td>\n",
       "      <td>0.154636</td>\n",
       "      <td>0.744230</td>\n",
       "      <td>228.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>max mean_per_class_accuracy</td>\n",
       "      <td>0.089810</td>\n",
       "      <td>0.758849</td>\n",
       "      <td>285.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>max tns</td>\n",
       "      <td>0.869474</td>\n",
       "      <td>144833.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>max fns</td>\n",
       "      <td>0.869474</td>\n",
       "      <td>15504.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>max fps</td>\n",
       "      <td>0.005437</td>\n",
       "      <td>144833.000000</td>\n",
       "      <td>399.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>max tps</td>\n",
       "      <td>0.005437</td>\n",
       "      <td>15505.000000</td>\n",
       "      <td>399.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>max tnr</td>\n",
       "      <td>0.869474</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>max fnr</td>\n",
       "      <td>0.869474</td>\n",
       "      <td>0.999936</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>max fpr</td>\n",
       "      <td>0.005437</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>399.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>max tpr</td>\n",
       "      <td>0.005437</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>399.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         metric  threshold          value    idx\n",
       "0                        max f1   0.172870       0.363668  212.0\n",
       "1                        max f2   0.108866       0.534668  269.0\n",
       "2                  max f0point5   0.257389       0.310773  130.0\n",
       "3                  max accuracy   0.451089       0.903622   38.0\n",
       "4                 max precision   0.869474       1.000000    0.0\n",
       "5                    max recall   0.005437       1.000000  399.0\n",
       "6               max specificity   0.869474       1.000000    0.0\n",
       "7              max absolute_mcc   0.126929       0.318723  252.0\n",
       "8    max min_per_class_accuracy   0.154636       0.744230  228.0\n",
       "9   max mean_per_class_accuracy   0.089810       0.758849  285.0\n",
       "10                      max tns   0.869474  144833.000000    0.0\n",
       "11                      max fns   0.869474   15504.000000    0.0\n",
       "12                      max fps   0.005437  144833.000000  399.0\n",
       "13                      max tps   0.005437   15505.000000  399.0\n",
       "14                      max tnr   0.869474       1.000000    0.0\n",
       "15                      max fnr   0.869474       0.999936    0.0\n",
       "16                      max fpr   0.005437       1.000000  399.0\n",
       "17                      max tpr   0.005437       1.000000  399.0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Gains/Lift Table: Avg response rate:  9.67 %, avg score:  9.68 %\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>group</th>\n",
       "      <th>cumulative_data_fraction</th>\n",
       "      <th>lower_threshold</th>\n",
       "      <th>lift</th>\n",
       "      <th>cumulative_lift</th>\n",
       "      <th>response_rate</th>\n",
       "      <th>score</th>\n",
       "      <th>cumulative_response_rate</th>\n",
       "      <th>cumulative_score</th>\n",
       "      <th>capture_rate</th>\n",
       "      <th>cumulative_capture_rate</th>\n",
       "      <th>gain</th>\n",
       "      <th>cumulative_gain</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "      <td>0.010029</td>\n",
       "      <td>0.368916</td>\n",
       "      <td>4.617459</td>\n",
       "      <td>4.617459</td>\n",
       "      <td>0.446517</td>\n",
       "      <td>0.428848</td>\n",
       "      <td>0.446517</td>\n",
       "      <td>0.428848</td>\n",
       "      <td>0.046308</td>\n",
       "      <td>0.046308</td>\n",
       "      <td>361.745946</td>\n",
       "      <td>361.745946</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td></td>\n",
       "      <td>2</td>\n",
       "      <td>0.020020</td>\n",
       "      <td>0.334800</td>\n",
       "      <td>3.679400</td>\n",
       "      <td>4.149307</td>\n",
       "      <td>0.355805</td>\n",
       "      <td>0.350178</td>\n",
       "      <td>0.401246</td>\n",
       "      <td>0.389586</td>\n",
       "      <td>0.036762</td>\n",
       "      <td>0.083070</td>\n",
       "      <td>267.940027</td>\n",
       "      <td>314.930655</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td></td>\n",
       "      <td>3</td>\n",
       "      <td>0.030030</td>\n",
       "      <td>0.312203</td>\n",
       "      <td>3.537219</td>\n",
       "      <td>3.945278</td>\n",
       "      <td>0.342056</td>\n",
       "      <td>0.323211</td>\n",
       "      <td>0.381516</td>\n",
       "      <td>0.367461</td>\n",
       "      <td>0.035408</td>\n",
       "      <td>0.118478</td>\n",
       "      <td>253.721941</td>\n",
       "      <td>294.527751</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td></td>\n",
       "      <td>4</td>\n",
       "      <td>0.040321</td>\n",
       "      <td>0.292185</td>\n",
       "      <td>3.171256</td>\n",
       "      <td>3.747731</td>\n",
       "      <td>0.306667</td>\n",
       "      <td>0.301728</td>\n",
       "      <td>0.362413</td>\n",
       "      <td>0.350685</td>\n",
       "      <td>0.032635</td>\n",
       "      <td>0.151113</td>\n",
       "      <td>217.125572</td>\n",
       "      <td>274.773134</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td></td>\n",
       "      <td>5</td>\n",
       "      <td>0.050026</td>\n",
       "      <td>0.280927</td>\n",
       "      <td>2.924205</td>\n",
       "      <td>3.587975</td>\n",
       "      <td>0.282776</td>\n",
       "      <td>0.285381</td>\n",
       "      <td>0.346964</td>\n",
       "      <td>0.338016</td>\n",
       "      <td>0.028378</td>\n",
       "      <td>0.179490</td>\n",
       "      <td>192.420473</td>\n",
       "      <td>258.797478</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td></td>\n",
       "      <td>6</td>\n",
       "      <td>0.100020</td>\n",
       "      <td>0.251331</td>\n",
       "      <td>2.749099</td>\n",
       "      <td>3.168668</td>\n",
       "      <td>0.265843</td>\n",
       "      <td>0.265267</td>\n",
       "      <td>0.306416</td>\n",
       "      <td>0.301653</td>\n",
       "      <td>0.137440</td>\n",
       "      <td>0.316930</td>\n",
       "      <td>174.909933</td>\n",
       "      <td>216.866783</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td></td>\n",
       "      <td>7</td>\n",
       "      <td>0.150077</td>\n",
       "      <td>0.222941</td>\n",
       "      <td>2.457063</td>\n",
       "      <td>2.931318</td>\n",
       "      <td>0.237603</td>\n",
       "      <td>0.237342</td>\n",
       "      <td>0.283464</td>\n",
       "      <td>0.280203</td>\n",
       "      <td>0.122993</td>\n",
       "      <td>0.439923</td>\n",
       "      <td>145.706264</td>\n",
       "      <td>193.131824</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td></td>\n",
       "      <td>8</td>\n",
       "      <td>0.200002</td>\n",
       "      <td>0.200541</td>\n",
       "      <td>2.261984</td>\n",
       "      <td>2.764235</td>\n",
       "      <td>0.218738</td>\n",
       "      <td>0.211146</td>\n",
       "      <td>0.267307</td>\n",
       "      <td>0.262965</td>\n",
       "      <td>0.112931</td>\n",
       "      <td>0.552854</td>\n",
       "      <td>126.198386</td>\n",
       "      <td>176.423511</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td></td>\n",
       "      <td>9</td>\n",
       "      <td>0.300016</td>\n",
       "      <td>0.155401</td>\n",
       "      <td>1.883650</td>\n",
       "      <td>2.470682</td>\n",
       "      <td>0.182153</td>\n",
       "      <td>0.178919</td>\n",
       "      <td>0.238920</td>\n",
       "      <td>0.234947</td>\n",
       "      <td>0.188391</td>\n",
       "      <td>0.741245</td>\n",
       "      <td>88.364996</td>\n",
       "      <td>147.068232</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td></td>\n",
       "      <td>10</td>\n",
       "      <td>0.400036</td>\n",
       "      <td>0.085962</td>\n",
       "      <td>1.256118</td>\n",
       "      <td>2.167008</td>\n",
       "      <td>0.121469</td>\n",
       "      <td>0.121000</td>\n",
       "      <td>0.209554</td>\n",
       "      <td>0.206457</td>\n",
       "      <td>0.125637</td>\n",
       "      <td>0.866882</td>\n",
       "      <td>25.611822</td>\n",
       "      <td>116.700816</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td></td>\n",
       "      <td>11</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.044132</td>\n",
       "      <td>0.672284</td>\n",
       "      <td>1.868172</td>\n",
       "      <td>0.065011</td>\n",
       "      <td>0.065191</td>\n",
       "      <td>0.180656</td>\n",
       "      <td>0.178214</td>\n",
       "      <td>0.067204</td>\n",
       "      <td>0.934086</td>\n",
       "      <td>-32.771553</td>\n",
       "      <td>86.817156</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td></td>\n",
       "      <td>12</td>\n",
       "      <td>0.600001</td>\n",
       "      <td>0.023740</td>\n",
       "      <td>0.336016</td>\n",
       "      <td>1.612810</td>\n",
       "      <td>0.032493</td>\n",
       "      <td>0.032241</td>\n",
       "      <td>0.155962</td>\n",
       "      <td>0.153885</td>\n",
       "      <td>0.033602</td>\n",
       "      <td>0.967688</td>\n",
       "      <td>-66.398355</td>\n",
       "      <td>61.280972</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td></td>\n",
       "      <td>13</td>\n",
       "      <td>0.700002</td>\n",
       "      <td>0.013173</td>\n",
       "      <td>0.161881</td>\n",
       "      <td>1.405532</td>\n",
       "      <td>0.015654</td>\n",
       "      <td>0.017749</td>\n",
       "      <td>0.135918</td>\n",
       "      <td>0.134437</td>\n",
       "      <td>0.016188</td>\n",
       "      <td>0.983876</td>\n",
       "      <td>-83.811876</td>\n",
       "      <td>40.553238</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td></td>\n",
       "      <td>14</td>\n",
       "      <td>0.800484</td>\n",
       "      <td>0.009717</td>\n",
       "      <td>0.079591</td>\n",
       "      <td>1.239092</td>\n",
       "      <td>0.007697</td>\n",
       "      <td>0.011207</td>\n",
       "      <td>0.119823</td>\n",
       "      <td>0.118968</td>\n",
       "      <td>0.007997</td>\n",
       "      <td>0.991874</td>\n",
       "      <td>-92.040902</td>\n",
       "      <td>23.909237</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td></td>\n",
       "      <td>15</td>\n",
       "      <td>0.900030</td>\n",
       "      <td>0.007618</td>\n",
       "      <td>0.051832</td>\n",
       "      <td>1.107778</td>\n",
       "      <td>0.005012</td>\n",
       "      <td>0.008573</td>\n",
       "      <td>0.107124</td>\n",
       "      <td>0.106758</td>\n",
       "      <td>0.005160</td>\n",
       "      <td>0.997033</td>\n",
       "      <td>-94.816840</td>\n",
       "      <td>10.777784</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td></td>\n",
       "      <td>16</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.004478</td>\n",
       "      <td>0.029677</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.002870</td>\n",
       "      <td>0.006722</td>\n",
       "      <td>0.096702</td>\n",
       "      <td>0.096758</td>\n",
       "      <td>0.002967</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-97.032327</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      group  cumulative_data_fraction  lower_threshold      lift  \\\n",
       "0         1                  0.010029         0.368916  4.617459   \n",
       "1         2                  0.020020         0.334800  3.679400   \n",
       "2         3                  0.030030         0.312203  3.537219   \n",
       "3         4                  0.040321         0.292185  3.171256   \n",
       "4         5                  0.050026         0.280927  2.924205   \n",
       "5         6                  0.100020         0.251331  2.749099   \n",
       "6         7                  0.150077         0.222941  2.457063   \n",
       "7         8                  0.200002         0.200541  2.261984   \n",
       "8         9                  0.300016         0.155401  1.883650   \n",
       "9        10                  0.400036         0.085962  1.256118   \n",
       "10       11                  0.500000         0.044132  0.672284   \n",
       "11       12                  0.600001         0.023740  0.336016   \n",
       "12       13                  0.700002         0.013173  0.161881   \n",
       "13       14                  0.800484         0.009717  0.079591   \n",
       "14       15                  0.900030         0.007618  0.051832   \n",
       "15       16                  1.000000         0.004478  0.029677   \n",
       "\n",
       "    cumulative_lift  response_rate     score  cumulative_response_rate  \\\n",
       "0          4.617459       0.446517  0.428848                  0.446517   \n",
       "1          4.149307       0.355805  0.350178                  0.401246   \n",
       "2          3.945278       0.342056  0.323211                  0.381516   \n",
       "3          3.747731       0.306667  0.301728                  0.362413   \n",
       "4          3.587975       0.282776  0.285381                  0.346964   \n",
       "5          3.168668       0.265843  0.265267                  0.306416   \n",
       "6          2.931318       0.237603  0.237342                  0.283464   \n",
       "7          2.764235       0.218738  0.211146                  0.267307   \n",
       "8          2.470682       0.182153  0.178919                  0.238920   \n",
       "9          2.167008       0.121469  0.121000                  0.209554   \n",
       "10         1.868172       0.065011  0.065191                  0.180656   \n",
       "11         1.612810       0.032493  0.032241                  0.155962   \n",
       "12         1.405532       0.015654  0.017749                  0.135918   \n",
       "13         1.239092       0.007697  0.011207                  0.119823   \n",
       "14         1.107778       0.005012  0.008573                  0.107124   \n",
       "15         1.000000       0.002870  0.006722                  0.096702   \n",
       "\n",
       "    cumulative_score  capture_rate  cumulative_capture_rate        gain  \\\n",
       "0           0.428848      0.046308                 0.046308  361.745946   \n",
       "1           0.389586      0.036762                 0.083070  267.940027   \n",
       "2           0.367461      0.035408                 0.118478  253.721941   \n",
       "3           0.350685      0.032635                 0.151113  217.125572   \n",
       "4           0.338016      0.028378                 0.179490  192.420473   \n",
       "5           0.301653      0.137440                 0.316930  174.909933   \n",
       "6           0.280203      0.122993                 0.439923  145.706264   \n",
       "7           0.262965      0.112931                 0.552854  126.198386   \n",
       "8           0.234947      0.188391                 0.741245   88.364996   \n",
       "9           0.206457      0.125637                 0.866882   25.611822   \n",
       "10          0.178214      0.067204                 0.934086  -32.771553   \n",
       "11          0.153885      0.033602                 0.967688  -66.398355   \n",
       "12          0.134437      0.016188                 0.983876  -83.811876   \n",
       "13          0.118968      0.007997                 0.991874  -92.040902   \n",
       "14          0.106758      0.005160                 0.997033  -94.816840   \n",
       "15          0.096758      0.002967                 1.000000  -97.032327   \n",
       "\n",
       "    cumulative_gain  \n",
       "0        361.745946  \n",
       "1        314.930655  \n",
       "2        294.527751  \n",
       "3        274.773134  \n",
       "4        258.797478  \n",
       "5        216.866783  \n",
       "6        193.131824  \n",
       "7        176.423511  \n",
       "8        147.068232  \n",
       "9        116.700816  \n",
       "10        86.817156  \n",
       "11        61.280972  \n",
       "12        40.553238  \n",
       "13        23.909237  \n",
       "14        10.777784  \n",
       "15         0.000000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Cross-Validation Metrics Summary: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>sd</th>\n",
       "      <th>cv_1_valid</th>\n",
       "      <th>cv_2_valid</th>\n",
       "      <th>cv_3_valid</th>\n",
       "      <th>cv_4_valid</th>\n",
       "      <th>cv_5_valid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>accuracy</td>\n",
       "      <td>0.7788916</td>\n",
       "      <td>0.022123443</td>\n",
       "      <td>0.79507786</td>\n",
       "      <td>0.80109996</td>\n",
       "      <td>0.7799413</td>\n",
       "      <td>0.7446169</td>\n",
       "      <td>0.77372193</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>auc</td>\n",
       "      <td>0.8200793</td>\n",
       "      <td>0.0018740107</td>\n",
       "      <td>0.81823957</td>\n",
       "      <td>0.8218998</td>\n",
       "      <td>0.81825435</td>\n",
       "      <td>0.822078</td>\n",
       "      <td>0.819925</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>err</td>\n",
       "      <td>0.2211084</td>\n",
       "      <td>0.022123443</td>\n",
       "      <td>0.20492214</td>\n",
       "      <td>0.19890006</td>\n",
       "      <td>0.22005866</td>\n",
       "      <td>0.2553831</td>\n",
       "      <td>0.22627805</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>err_count</td>\n",
       "      <td>7093.4</td>\n",
       "      <td>748.045</td>\n",
       "      <td>6553.0</td>\n",
       "      <td>6329.0</td>\n",
       "      <td>7052.0</td>\n",
       "      <td>8243.0</td>\n",
       "      <td>7290.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>f0point5</td>\n",
       "      <td>0.28908145</td>\n",
       "      <td>0.013466386</td>\n",
       "      <td>0.29325423</td>\n",
       "      <td>0.3072256</td>\n",
       "      <td>0.28294584</td>\n",
       "      <td>0.27073926</td>\n",
       "      <td>0.29124236</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>f1</td>\n",
       "      <td>0.3650672</td>\n",
       "      <td>0.008101144</td>\n",
       "      <td>0.36434183</td>\n",
       "      <td>0.37565354</td>\n",
       "      <td>0.3589091</td>\n",
       "      <td>0.35596532</td>\n",
       "      <td>0.37046632</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>f2</td>\n",
       "      <td>0.4966489</td>\n",
       "      <td>0.016836906</td>\n",
       "      <td>0.4809219</td>\n",
       "      <td>0.4832978</td>\n",
       "      <td>0.49062982</td>\n",
       "      <td>0.5194983</td>\n",
       "      <td>0.50889677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>lift_top_group</td>\n",
       "      <td>4.653895</td>\n",
       "      <td>0.15403096</td>\n",
       "      <td>4.6264467</td>\n",
       "      <td>4.558055</td>\n",
       "      <td>4.9188175</td>\n",
       "      <td>4.631808</td>\n",
       "      <td>4.534347</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>logloss</td>\n",
       "      <td>0.25483617</td>\n",
       "      <td>0.004168726</td>\n",
       "      <td>0.25488704</td>\n",
       "      <td>0.25988072</td>\n",
       "      <td>0.2522701</td>\n",
       "      <td>0.24943149</td>\n",
       "      <td>0.25771156</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>max_per_class_error</td>\n",
       "      <td>0.34395748</td>\n",
       "      <td>0.05855545</td>\n",
       "      <td>0.38867188</td>\n",
       "      <td>0.40257296</td>\n",
       "      <td>0.35044423</td>\n",
       "      <td>0.2558235</td>\n",
       "      <td>0.3222749</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>mcc</td>\n",
       "      <td>0.30662048</td>\n",
       "      <td>0.006561466</td>\n",
       "      <td>0.29995707</td>\n",
       "      <td>0.30604073</td>\n",
       "      <td>0.30048138</td>\n",
       "      <td>0.3136596</td>\n",
       "      <td>0.3129637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>mean_per_class_accuracy</td>\n",
       "      <td>0.72452193</td>\n",
       "      <td>0.014671418</td>\n",
       "      <td>0.71296704</td>\n",
       "      <td>0.7105984</td>\n",
       "      <td>0.72157866</td>\n",
       "      <td>0.74651295</td>\n",
       "      <td>0.7309526</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>mean_per_class_error</td>\n",
       "      <td>0.27547807</td>\n",
       "      <td>0.014671418</td>\n",
       "      <td>0.28703296</td>\n",
       "      <td>0.2894016</td>\n",
       "      <td>0.27842134</td>\n",
       "      <td>0.25348702</td>\n",
       "      <td>0.2690474</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>mse</td>\n",
       "      <td>0.07631457</td>\n",
       "      <td>0.0014433885</td>\n",
       "      <td>0.076081716</td>\n",
       "      <td>0.078355685</td>\n",
       "      <td>0.07526237</td>\n",
       "      <td>0.07477031</td>\n",
       "      <td>0.07710276</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>precision</td>\n",
       "      <td>0.25395817</td>\n",
       "      <td>0.014892657</td>\n",
       "      <td>0.2594998</td>\n",
       "      <td>0.27395684</td>\n",
       "      <td>0.2479588</td>\n",
       "      <td>0.2334734</td>\n",
       "      <td>0.25490198</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>r2</td>\n",
       "      <td>0.12628643</td>\n",
       "      <td>0.0035357908</td>\n",
       "      <td>0.123859614</td>\n",
       "      <td>0.1305954</td>\n",
       "      <td>0.12321721</td>\n",
       "      <td>0.12410273</td>\n",
       "      <td>0.12965718</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>recall</td>\n",
       "      <td>0.6569771</td>\n",
       "      <td>0.060324375</td>\n",
       "      <td>0.6113281</td>\n",
       "      <td>0.59742707</td>\n",
       "      <td>0.6495558</td>\n",
       "      <td>0.74884945</td>\n",
       "      <td>0.67772514</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>rmse</td>\n",
       "      <td>0.27624106</td>\n",
       "      <td>0.0026083938</td>\n",
       "      <td>0.27582914</td>\n",
       "      <td>0.27992085</td>\n",
       "      <td>0.27433988</td>\n",
       "      <td>0.27344158</td>\n",
       "      <td>0.27767384</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>specificity</td>\n",
       "      <td>0.79206675</td>\n",
       "      <td>0.031109199</td>\n",
       "      <td>0.81460595</td>\n",
       "      <td>0.82376975</td>\n",
       "      <td>0.7936016</td>\n",
       "      <td>0.7441765</td>\n",
       "      <td>0.7841801</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   mean            sd   cv_1_valid  \\\n",
       "0                  accuracy   0.7788916   0.022123443   0.79507786   \n",
       "1                       auc   0.8200793  0.0018740107   0.81823957   \n",
       "2                       err   0.2211084   0.022123443   0.20492214   \n",
       "3                 err_count      7093.4       748.045       6553.0   \n",
       "4                  f0point5  0.28908145   0.013466386   0.29325423   \n",
       "5                        f1   0.3650672   0.008101144   0.36434183   \n",
       "6                        f2   0.4966489   0.016836906    0.4809219   \n",
       "7            lift_top_group    4.653895    0.15403096    4.6264467   \n",
       "8                   logloss  0.25483617   0.004168726   0.25488704   \n",
       "9       max_per_class_error  0.34395748    0.05855545   0.38867188   \n",
       "10                      mcc  0.30662048   0.006561466   0.29995707   \n",
       "11  mean_per_class_accuracy  0.72452193   0.014671418   0.71296704   \n",
       "12     mean_per_class_error  0.27547807   0.014671418   0.28703296   \n",
       "13                      mse  0.07631457  0.0014433885  0.076081716   \n",
       "14                precision  0.25395817   0.014892657    0.2594998   \n",
       "15                       r2  0.12628643  0.0035357908  0.123859614   \n",
       "16                   recall   0.6569771   0.060324375    0.6113281   \n",
       "17                     rmse  0.27624106  0.0026083938   0.27582914   \n",
       "18              specificity  0.79206675   0.031109199   0.81460595   \n",
       "\n",
       "     cv_2_valid  cv_3_valid  cv_4_valid  cv_5_valid  \n",
       "0    0.80109996   0.7799413   0.7446169  0.77372193  \n",
       "1     0.8218998  0.81825435    0.822078    0.819925  \n",
       "2    0.19890006  0.22005866   0.2553831  0.22627805  \n",
       "3        6329.0      7052.0      8243.0      7290.0  \n",
       "4     0.3072256  0.28294584  0.27073926  0.29124236  \n",
       "5    0.37565354   0.3589091  0.35596532  0.37046632  \n",
       "6     0.4832978  0.49062982   0.5194983  0.50889677  \n",
       "7      4.558055   4.9188175    4.631808    4.534347  \n",
       "8    0.25988072   0.2522701  0.24943149  0.25771156  \n",
       "9    0.40257296  0.35044423   0.2558235   0.3222749  \n",
       "10   0.30604073  0.30048138   0.3136596   0.3129637  \n",
       "11    0.7105984  0.72157866  0.74651295   0.7309526  \n",
       "12    0.2894016  0.27842134  0.25348702   0.2690474  \n",
       "13  0.078355685  0.07526237  0.07477031  0.07710276  \n",
       "14   0.27395684   0.2479588   0.2334734  0.25490198  \n",
       "15    0.1305954  0.12321721  0.12410273  0.12965718  \n",
       "16   0.59742707   0.6495558  0.74884945  0.67772514  \n",
       "17   0.27992085  0.27433988  0.27344158  0.27767384  \n",
       "18   0.82376975   0.7936016   0.7441765   0.7841801  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Scoring History: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>timestamp</th>\n",
       "      <th>duration</th>\n",
       "      <th>number_of_trees</th>\n",
       "      <th>training_rmse</th>\n",
       "      <th>training_logloss</th>\n",
       "      <th>training_auc</th>\n",
       "      <th>training_pr_auc</th>\n",
       "      <th>training_lift</th>\n",
       "      <th>training_classification_error</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:20:52</td>\n",
       "      <td>13.272 sec</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.295552</td>\n",
       "      <td>0.317775</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.903298</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:20:52</td>\n",
       "      <td>13.304 sec</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.292387</td>\n",
       "      <td>0.307527</td>\n",
       "      <td>0.798066</td>\n",
       "      <td>0.261304</td>\n",
       "      <td>4.197561</td>\n",
       "      <td>0.253939</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:20:52</td>\n",
       "      <td>13.328 sec</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.289790</td>\n",
       "      <td>0.299591</td>\n",
       "      <td>0.802684</td>\n",
       "      <td>0.267325</td>\n",
       "      <td>4.296634</td>\n",
       "      <td>0.272474</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:20:52</td>\n",
       "      <td>13.355 sec</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.287696</td>\n",
       "      <td>0.293356</td>\n",
       "      <td>0.805310</td>\n",
       "      <td>0.269337</td>\n",
       "      <td>4.301683</td>\n",
       "      <td>0.217509</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:20:52</td>\n",
       "      <td>13.379 sec</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.285985</td>\n",
       "      <td>0.288267</td>\n",
       "      <td>0.806769</td>\n",
       "      <td>0.270388</td>\n",
       "      <td>4.167028</td>\n",
       "      <td>0.216786</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:20:52</td>\n",
       "      <td>13.403 sec</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.284534</td>\n",
       "      <td>0.283930</td>\n",
       "      <td>0.808244</td>\n",
       "      <td>0.271697</td>\n",
       "      <td>4.294578</td>\n",
       "      <td>0.215158</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:20:52</td>\n",
       "      <td>13.426 sec</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.283368</td>\n",
       "      <td>0.280415</td>\n",
       "      <td>0.808974</td>\n",
       "      <td>0.273015</td>\n",
       "      <td>4.278063</td>\n",
       "      <td>0.233288</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:20:52</td>\n",
       "      <td>13.452 sec</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.282369</td>\n",
       "      <td>0.277301</td>\n",
       "      <td>0.809773</td>\n",
       "      <td>0.274218</td>\n",
       "      <td>4.431879</td>\n",
       "      <td>0.234056</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:20:52</td>\n",
       "      <td>13.479 sec</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.281528</td>\n",
       "      <td>0.274647</td>\n",
       "      <td>0.810421</td>\n",
       "      <td>0.275448</td>\n",
       "      <td>4.452931</td>\n",
       "      <td>0.230083</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:20:52</td>\n",
       "      <td>13.511 sec</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.280854</td>\n",
       "      <td>0.272468</td>\n",
       "      <td>0.810525</td>\n",
       "      <td>0.275984</td>\n",
       "      <td>4.463115</td>\n",
       "      <td>0.231461</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:20:52</td>\n",
       "      <td>13.545 sec</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.280277</td>\n",
       "      <td>0.270525</td>\n",
       "      <td>0.810870</td>\n",
       "      <td>0.276660</td>\n",
       "      <td>4.483620</td>\n",
       "      <td>0.231268</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:20:52</td>\n",
       "      <td>13.580 sec</td>\n",
       "      <td>11.0</td>\n",
       "      <td>0.279791</td>\n",
       "      <td>0.268860</td>\n",
       "      <td>0.811251</td>\n",
       "      <td>0.277438</td>\n",
       "      <td>4.515021</td>\n",
       "      <td>0.233582</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:20:52</td>\n",
       "      <td>13.619 sec</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.279347</td>\n",
       "      <td>0.267279</td>\n",
       "      <td>0.811904</td>\n",
       "      <td>0.278154</td>\n",
       "      <td>4.503857</td>\n",
       "      <td>0.237037</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:20:52</td>\n",
       "      <td>13.657 sec</td>\n",
       "      <td>13.0</td>\n",
       "      <td>0.278989</td>\n",
       "      <td>0.265942</td>\n",
       "      <td>0.812410</td>\n",
       "      <td>0.278936</td>\n",
       "      <td>4.498789</td>\n",
       "      <td>0.241845</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:20:52</td>\n",
       "      <td>13.696 sec</td>\n",
       "      <td>14.0</td>\n",
       "      <td>0.278705</td>\n",
       "      <td>0.264840</td>\n",
       "      <td>0.812517</td>\n",
       "      <td>0.279414</td>\n",
       "      <td>4.579063</td>\n",
       "      <td>0.234155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:20:52</td>\n",
       "      <td>13.735 sec</td>\n",
       "      <td>15.0</td>\n",
       "      <td>0.278439</td>\n",
       "      <td>0.263808</td>\n",
       "      <td>0.812936</td>\n",
       "      <td>0.280343</td>\n",
       "      <td>4.590324</td>\n",
       "      <td>0.233819</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:20:52</td>\n",
       "      <td>13.775 sec</td>\n",
       "      <td>16.0</td>\n",
       "      <td>0.278045</td>\n",
       "      <td>0.262597</td>\n",
       "      <td>0.814427</td>\n",
       "      <td>0.283438</td>\n",
       "      <td>4.494780</td>\n",
       "      <td>0.231642</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:20:52</td>\n",
       "      <td>13.819 sec</td>\n",
       "      <td>17.0</td>\n",
       "      <td>0.277758</td>\n",
       "      <td>0.261621</td>\n",
       "      <td>0.815323</td>\n",
       "      <td>0.285511</td>\n",
       "      <td>4.669160</td>\n",
       "      <td>0.226010</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:20:52</td>\n",
       "      <td>13.866 sec</td>\n",
       "      <td>18.0</td>\n",
       "      <td>0.277516</td>\n",
       "      <td>0.260770</td>\n",
       "      <td>0.816112</td>\n",
       "      <td>0.287034</td>\n",
       "      <td>4.708927</td>\n",
       "      <td>0.226260</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:20:52</td>\n",
       "      <td>13.911 sec</td>\n",
       "      <td>19.0</td>\n",
       "      <td>0.277287</td>\n",
       "      <td>0.259982</td>\n",
       "      <td>0.816809</td>\n",
       "      <td>0.288302</td>\n",
       "      <td>4.732450</td>\n",
       "      <td>0.225549</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                timestamp    duration  number_of_trees  training_rmse  \\\n",
       "0     2019-12-09 13:20:52  13.272 sec              0.0       0.295552   \n",
       "1     2019-12-09 13:20:52  13.304 sec              1.0       0.292387   \n",
       "2     2019-12-09 13:20:52  13.328 sec              2.0       0.289790   \n",
       "3     2019-12-09 13:20:52  13.355 sec              3.0       0.287696   \n",
       "4     2019-12-09 13:20:52  13.379 sec              4.0       0.285985   \n",
       "5     2019-12-09 13:20:52  13.403 sec              5.0       0.284534   \n",
       "6     2019-12-09 13:20:52  13.426 sec              6.0       0.283368   \n",
       "7     2019-12-09 13:20:52  13.452 sec              7.0       0.282369   \n",
       "8     2019-12-09 13:20:52  13.479 sec              8.0       0.281528   \n",
       "9     2019-12-09 13:20:52  13.511 sec              9.0       0.280854   \n",
       "10    2019-12-09 13:20:52  13.545 sec             10.0       0.280277   \n",
       "11    2019-12-09 13:20:52  13.580 sec             11.0       0.279791   \n",
       "12    2019-12-09 13:20:52  13.619 sec             12.0       0.279347   \n",
       "13    2019-12-09 13:20:52  13.657 sec             13.0       0.278989   \n",
       "14    2019-12-09 13:20:52  13.696 sec             14.0       0.278705   \n",
       "15    2019-12-09 13:20:52  13.735 sec             15.0       0.278439   \n",
       "16    2019-12-09 13:20:52  13.775 sec             16.0       0.278045   \n",
       "17    2019-12-09 13:20:52  13.819 sec             17.0       0.277758   \n",
       "18    2019-12-09 13:20:52  13.866 sec             18.0       0.277516   \n",
       "19    2019-12-09 13:20:52  13.911 sec             19.0       0.277287   \n",
       "\n",
       "    training_logloss  training_auc  training_pr_auc  training_lift  \\\n",
       "0           0.317775      0.500000         0.000000       1.000000   \n",
       "1           0.307527      0.798066         0.261304       4.197561   \n",
       "2           0.299591      0.802684         0.267325       4.296634   \n",
       "3           0.293356      0.805310         0.269337       4.301683   \n",
       "4           0.288267      0.806769         0.270388       4.167028   \n",
       "5           0.283930      0.808244         0.271697       4.294578   \n",
       "6           0.280415      0.808974         0.273015       4.278063   \n",
       "7           0.277301      0.809773         0.274218       4.431879   \n",
       "8           0.274647      0.810421         0.275448       4.452931   \n",
       "9           0.272468      0.810525         0.275984       4.463115   \n",
       "10          0.270525      0.810870         0.276660       4.483620   \n",
       "11          0.268860      0.811251         0.277438       4.515021   \n",
       "12          0.267279      0.811904         0.278154       4.503857   \n",
       "13          0.265942      0.812410         0.278936       4.498789   \n",
       "14          0.264840      0.812517         0.279414       4.579063   \n",
       "15          0.263808      0.812936         0.280343       4.590324   \n",
       "16          0.262597      0.814427         0.283438       4.494780   \n",
       "17          0.261621      0.815323         0.285511       4.669160   \n",
       "18          0.260770      0.816112         0.287034       4.708927   \n",
       "19          0.259982      0.816809         0.288302       4.732450   \n",
       "\n",
       "    training_classification_error  \n",
       "0                        0.903298  \n",
       "1                        0.253939  \n",
       "2                        0.272474  \n",
       "3                        0.217509  \n",
       "4                        0.216786  \n",
       "5                        0.215158  \n",
       "6                        0.233288  \n",
       "7                        0.234056  \n",
       "8                        0.230083  \n",
       "9                        0.231461  \n",
       "10                       0.231268  \n",
       "11                       0.233582  \n",
       "12                       0.237037  \n",
       "13                       0.241845  \n",
       "14                       0.234155  \n",
       "15                       0.233819  \n",
       "16                       0.231642  \n",
       "17                       0.226010  \n",
       "18                       0.226260  \n",
       "19                       0.225549  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "See the whole table with table.as_data_frame()\n",
      "\n",
      "Variable Importances: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>variable</th>\n",
       "      <th>relative_importance</th>\n",
       "      <th>scaled_importance</th>\n",
       "      <th>percentage</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>loan_to_value_ratio_std</td>\n",
       "      <td>4938.643066</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.523122</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>property_value_std</td>\n",
       "      <td>1754.421265</td>\n",
       "      <td>0.355244</td>\n",
       "      <td>0.185836</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>debt_to_income_ratio_std</td>\n",
       "      <td>1225.825439</td>\n",
       "      <td>0.248211</td>\n",
       "      <td>0.129845</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>intro_rate_period_std</td>\n",
       "      <td>635.243591</td>\n",
       "      <td>0.128627</td>\n",
       "      <td>0.067288</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>loan_amount_std</td>\n",
       "      <td>521.774658</td>\n",
       "      <td>0.105651</td>\n",
       "      <td>0.055269</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>no_intro_rate_period_std</td>\n",
       "      <td>156.612869</td>\n",
       "      <td>0.031712</td>\n",
       "      <td>0.016589</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>income_std</td>\n",
       "      <td>119.916946</td>\n",
       "      <td>0.024281</td>\n",
       "      <td>0.012702</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>term_360</td>\n",
       "      <td>47.980827</td>\n",
       "      <td>0.009715</td>\n",
       "      <td>0.005082</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>conforming</td>\n",
       "      <td>33.125870</td>\n",
       "      <td>0.006707</td>\n",
       "      <td>0.003509</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>debt_to_income_ratio_missing.False</td>\n",
       "      <td>7.173559</td>\n",
       "      <td>0.001453</td>\n",
       "      <td>0.000760</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>debt_to_income_ratio_missing.True</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>debt_to_income_ratio_missing.missing(NA)</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    variable  relative_importance  \\\n",
       "0                    loan_to_value_ratio_std          4938.643066   \n",
       "1                         property_value_std          1754.421265   \n",
       "2                   debt_to_income_ratio_std          1225.825439   \n",
       "3                      intro_rate_period_std           635.243591   \n",
       "4                            loan_amount_std           521.774658   \n",
       "5                   no_intro_rate_period_std           156.612869   \n",
       "6                                 income_std           119.916946   \n",
       "7                                   term_360            47.980827   \n",
       "8                                 conforming            33.125870   \n",
       "9         debt_to_income_ratio_missing.False             7.173559   \n",
       "10         debt_to_income_ratio_missing.True             0.000000   \n",
       "11  debt_to_income_ratio_missing.missing(NA)             0.000000   \n",
       "\n",
       "    scaled_importance  percentage  \n",
       "0            1.000000    0.523122  \n",
       "1            0.355244    0.185836  \n",
       "2            0.248211    0.129845  \n",
       "3            0.128627    0.067288  \n",
       "4            0.105651    0.055269  \n",
       "5            0.031712    0.016589  \n",
       "6            0.024281    0.012702  \n",
       "7            0.009715    0.005082  \n",
       "8            0.006707    0.003509  \n",
       "9            0.001453    0.000760  \n",
       "10           0.000000    0.000000  \n",
       "11           0.000000    0.000000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": []
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gbm_default"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8237209254218628\n"
     ]
    }
   ],
   "source": [
    "# Get the AUC on the test set for the default GBM\n",
    "perf_gbm_default = gbm_default.model_performance(test)\n",
    "print(perf_gbm_default.auc())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8198539766244762\n"
     ]
    }
   ],
   "source": [
    "# Get the AUC on 5 fold CV for the default GBM\n",
    "perf_cv_gbm_default = gbm_default.model_performance(xval=True)\n",
    "print(perf_cv_gbm_default.auc())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Variable Importance of MGBM and GBM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA9oAAAJTCAYAAAD3327jAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3XmYZFV9//H3BwZZFRTRIEbHIBEXcGRzARQU13FDQaJGBI0o7rsoSQR/GsaQaMQFBEVAjRpUFEFA2TdlkWUGBdHAaBTFFUQ2Zfj+/rinoSiqunuYO9Mz8H49Tz9Vde6555x7qxvmU+fcW6kqJEmSJElSP1aa6QFIkiRJknR3YtCWJEmSJKlHBm1JkiRJknpk0JYkSZIkqUcGbUmSJEmSemTQliRJkiSpRwZtSZLugiRnJrmlh3Z+keSni1H/4UkqyWeWtG9JkrR0GLQlSSuEJF9sAfP106j7nVZ3x2Uxtrub9iFCJdlmpseytC3uBx33BEl2aO//iZPUmfjA56dD5Q9O8pYkxydZmOTmJL9vf5MvnKLfVZL8U5LvJvlNkr+0x+8keVWSWUtwTE9LckSSnyS5rrX96yQnJnlPkg1G7DPxdzD4c0uSq5Mck+SZI/aZNVB3UZKHTjKmMwbq/uNdPTZJy6e7/B8sSZKWsUOAlwH/BHxqXKUks4EdgF8B31qK43kZsPpSbF9aEb0VeAdwBXAycDUwG9gReHqS/avq3cM7JXkIcDTwWODXwDHt8W+A5wBPB16f5PlVddV0B5NkHeBw4PnAX4DTW9s3AOsBjwfmAfsm2aqq5o9o5nPAz9vz1YFHAs8G5iZ5dVUdOmKfW+j+nf1q4F9HjGtjYJuBepLuZvzDliStEKrq1CSXA49LsllVXTCm6quBAJ+rqiVe2j3JeH4+dS3pHuf7wJOr6ozBwiSPAc4G3pXki1V18cC2tYDj6QLsocAbq+rGge1rAgcB/wh8O8kTB7eP02bAvw5sTxf6X1lVvxhR7zHAB4D7jGnq0Ko6c2ifXYAvA+9rYx72S+CPwKuS7FtVi4a2v6Y9HgNMOtMvacXk0nFJ0orkkPb4mlEbk6wM7A4U8JmB8g2SvD/J2W256F+S/LItR994RDu3XQed5BFJjkzy2yS3TiynHnWNdpJVk7wpyXFJftaWzf6hLYW90zLToX3XSfKpJFcluSnJD5O8IUmme3KSrJnkfUkuTnJ9kj+3Y95lum1M0f4vkvw0yX2SfKy9vjHJhUme3+rMSvIvbYnuTa3+niPamlie/M9Jtk5yUpI/tZ/jkmw2ZgzrJPnwQPt/SLdM+alT9PGEJN9u9SvJPyYpYANgw6HlwYO/Oy9qvyc/GTin5yd5Y5I7/TsqyRdaG3+b5PVJLmnj/HWSg5KMDHOt/scHjuv3Sc5NsveYup9KckVuX5r9zSSbT/b+LQtV9dXhkN3KLwG+2l5uN7T5nXQh+3Tgn4ZDdFVdD+wGnEM34/3maQ5nV7qQfRnw3FEhe2JsVfUiug8Jpus77XG9SeocQvf79ezBwiT3amM7HfjxYvQpaQVi0JYkrUgOp1v++dIka4zY/my6f9ieWFVXDpRvD7wb+APwNeC/gHOBlwDnthmtUf6+1Xsw8AW6fzhfN8n41mttrwV8F/gI3XLYzYHjkuw2Zr9V6WbcdgD+u/WzLvCJ1t6UktwXOAv4EPBXulm2w4EHAl9Oss902pmGVYETgWcC36A7LxsBX0+yHd353QM4Bfgs3Szhp5K8eEx7T2p1b6Q73hOAZwBnJnnS0DHeD/get7+X/wUcBWwNnJjkn8b0sQ1dqLlXG9MRwE+Afenezz+25xM/Rw/s++/AHLoQ9nHg8+2YPt7aGuc/6d6LC4FP0i2Dfi3d+bmDJI8HLgbeCPwC+BjwJeDPDC07TrIFcBHwOroAeQDdJRLbAWcnecZQ/Ylrhpfa6o7F8Nf2ODyWiQ/OPlhVNWrHNiP8b+3lHtPsb6Ldf5/ODPhiroDZoT2eP0mdL9ItUR/+vdwRuD+3f3Ao6e6oqvzxxx9//PFnhfkBvkI3Y73biG3fbNt2Gip/ILDWiPqPA64HvjVU/vDWTgEfGDOOM4FbhspWAzYYUXcd4FLgt8CqQ9t+0fo5DbjXQPn9gSvbtieNGNtnhtr5Qit/+1D56nSh/1Zgk2me4zNbW9uMGes3Bo+D7oOMogu/3wfWHti2EV3AOm+orR0GzvHrhra9uJVfBmSg/LOt/FND9TemC8w3AX87po9XjznWXwA/neRcbDiibCW6EFXA5mPehyuBBw+Ur0K3dLqAzQbKV6W7/reAl4zoa7iNK+g+lBh+bx5Md1+CXwz9Hs1qbd8y7hhH9Dlx3q4A9hnzc0CrM/bcjfgb+C2wCNhooPxhrZ2/MPS3MaKNtdr+BfzNFHXvRRfoC3jodI99zN/BoQPH/eH2+/8XYAGw8dA+E+d7YXt9WPv9X3+gzol0fyur0V0fXsA/3pUx+uOPP8vvjzPakqQVzcHt8Q6zREnWp7tp0m/oAvdtqurqqvrzcENVdSFdwH1aumXnw64CPjjdgVXVTVX1yxHl19DdUOn+dLPbo+xVVX8Z2Od3dDOi0C2HHyvJA4CXAt+vqo8M9X0jsBfddesvneahTOUtVXXzQB+nAP8H3Bd4d1VdO7DtJ3The9Mxy+AvAz49NOav0c3OP4Juxpskq9LdgO5PdNfFDta/jG42fFXgFSP6OL+qJpt9Hquq/ndE2a10s87QzeyPsm8NLFWuqr/S/Q4AbDVQ74XA3wJfr6r/GdHX4HLn59MF0/+qoWuGW73/oFvRsd1A+S10y7IfPWack3kY8P4xP2+abiPtfT+U7vf/E+13YsL67fE3g79To7S/4T+2lw+aotv7AxN/03f6m0zy1CT7DP08f0xbu3P7cb8beAHd7+F/030YMZlD6ML37q3fvwOeCnyhqm6aYl9JKzBvhiZJWtGcDPwvsHWSR1bVpa18d7r/rx3WQs0dtH9Ev5Yu6K7Lnf8feD+6GbdBFw2G3+lIsgnwLrrlyg+iC3+D7vQ1QsDNjL4+9NT2+Lgput2KbpY1Y5aIT4zhkVO0Mx2/q6qfjSi/ii4wjrpJ3S/pZhgfQHcX6kFnVNWo5cKn0i0Jfxxd6H4U3QzgOe2Di2En032gMOpcnTuibFqS3J/u/XwOXfBcc6jKqPcTRi8p/r/2eN+Bsie0x+OmMZwntseHjXmfH9EeH8nt1xBPfBBxV5xUVTuM2pDk4XTL76fjY3TLpU+lO5fLg6cCw9e/f5Y7XjYwYduJDzba9dWzgbfRLWV/RpKntQ9f7qSqzkpyKfDqJPvRLWcPLhuX7vYM2pKkFUpVTdysaj+6We13tBmzV9MtwbzTP2CTvINutu8PdMs2f0a3/LaAFwGbcOdADN11tdOWZOvW/krASXQz69fRLdveDHjemH5+OyZsTvS/9hRdr9seH99+xllrinam49ox5bcAi0atHOD2a3JXGbFtOHhPGD72icdfjak/Ub7OJG0tlnZN+PnAQ+luxHUE3e/QLXQfzLyJ0e8nwKgPAybOw+DqiYnx3mnWdYSJ93mqm9v18T73IslH6M7TKXQ3JBv+4GrivVkvyaqTzWqnu/v4xIcUU33F1+/olpmvTPeB1x2+JaCq/hn459bus5jeBx208V8O7JnkcXSrB14MHDnJbofQ3a/hmbSbulXVgun0J2nFZdCWJK2IPkf3dTy7JnkvsC3wd8DJVfXTwYpJVqFb8nkV3bWxVw9t33aSfkbemGkS/0I367rt8NLeJP9CF7RHWS9JRoTtv2mP48ItQ9tHfkfxcu6BY8qHj/3aofJh6w/VG7S47+OEPehC9r9U1R0uIWi/N9NePj2JiUA+bmZ80MSxza2qb/fQ91LTPvw6gO4GbycCz68RNySrqiuS/Iru/Xsy3f0Exnkq3YdYV1TVpB+eVNVfkpxHt2Lgady+bL9P59B9sLUVkwftI+g+GDyE7vf3TneSl3T34zXakqQVTgvLR9Ndh/lCbr9e++AR1R8I3Bs4c0TIvg9TL8teHA+nu9b0zBHbnjLJfqty+xLiQdu1xwun6PccujA52YcGy6ttx1y7vV17nDj2S+mW2D9uzFdkbd8ex32/+jgTs56jPLw93ulO4Uz+fi6OiUsGnj1prTvWXa7f5/Z+HkQXso8HnjcqZA+Y+Dq19435XaB9ldrEtfmj/s4na/ddSVab5j6LY2J2fdJ/T1fV7+m+z/vBdCtcvrIUxiJpOWPQliStqCaWiL+D7vrP39F91dOwX9EFtC3b0lPgtmstP84dr5ddUgvpZqfvcOOpJK+lm1WbzLw2pol97s/tM1+TzsZV1a+ALwNPSPLeUTd2S/fd4A+d+hCWuY3prp2/TfsqsK3pvmP4bOhuNEd386m16VYzDNbfiC7U/YXurt+L4/fAA9rN1oYtbI/bDfW3BfCexexnnG/QXbv9oiQ7D29M8uCBl0e1Mb05Y76XPcmThkNlko2TPGJU/b61QPxZutUAxwAvnMZNv/ane6+3Az49Yvxr0N1M7Ql0X4N2wDSHczjddeGPBL6VZNyqgVGXG0yq3dTsBe3lqdPY5b10/516VnXfCy7pbs6l45KkFdV36ELHxB2cPzHqxmVVtSjJx4F3AguSHE03g/xUutB2Gv3NTn6ULlCfneR/6O5MvBXdTay+Rnct5yi/oJt1v2RgfDvRLTM9oKrOnkbfe9LNwP4bsFuSM+lu7rY+3Y3EtgB2prs+fXlyHHBAkrl0X5e0Ed118zfSfSXX4LLviZvMvSXJVnTv3Xp034e+FrBnVd3hWtxpOIluVcPxSc6gC+sXVtWxdF/N9A7g40l2AH5K993qz6V7P6e6VnpKVXVzC9jHA/+T5BS6m7etThcQn0x3OcJE3Re1uscnOYvuO7VvBB4CbEl3w7b16L7qjCSz6FYDLGLZ/LtvX7obE94AzAfeO2KS+oKquu2mY1V1XbtO+mi6m4U9N8lxdNdv/w0wl25lygVMPTt+m6q6JcmOdN99/lzgiiSnAT9s41sPeAzd3+fNdCtDRnlVe/+hu8/AbLqVNGsAR1XVt6Yxlp+x/P3tSVqKDNqSpBXSwE3RJq6dnewuvu+l+9qvV9HNnl5Ddy3o3nTXTvY1pmOTvKC1+w90N786l26mbmPGB+2b6YL/fnRfYbUu3Z3VPwR8cpp9X9uuG34t3dd47UQX2K+muzv0W+nuzL28OZvuOP8ft1/z/F1g76r6wWDFqvp9ksfTLSHeEXg7XWD6Ht316Sfehf73Be5DF8S2pVtG/lng2Kr6RTun8+gC77PoQutrgdPpIWgDVNU5SebQ/Z4+i242/zq6YL/PUN0Lk2xKd+zPpfudvpVu5cYP6O4T8EdmzsPa4xoMfQ3bgDvd3buqFraVArvRndfn0800X0P3YcLewOHt68qmrd2h/nlJng7sSheqt6YLzH+gC93vBT4/6qv5msGv1yu6a+V/QHft9aGLMx5J9xwZfZNTSZKkpafNEH6XETcakyRpRec12pIkSZIk9cigLUmSJElSjwzakiRJkiT1yGu0JUmSJEnqkXcdl4DDDz+8XvnKV870MCRJkiQtv+70fYXjuHRcAq6//vqZHoIkSZKkuwmDtiRJkiRJPTJoS5IkSZLUI4O2JEmSJEk9MmhLkiRJktQjg7YkSZIkST0yaEuSJEmS1CODtiRJkiRJPTJoS5IkSZLUI4O2JEmSJEk9MmhLkiRJktQjg7YkSZIkST0yaEuSJEmS1CODtiRJkiRJPTJoS5IkSZLUI4O2JEmSJEk9MmhLkiRJktQjg7YkSZIkST0yaEuSJEmS1CODtiRJkiRJPTJoS5IkSZLUI4O2JEmSJEk9MmhLkiRJktQjg7YkSZIkST0yaEuSJEmS1CODtiRJkiRJPTJoS5IkSZLUI4O2JEmSJEk9MmhLkiRJktSjWTM9AGl5sOCX1zJ7r2NnehiSJEmSgIXz5s70EJaIM9qSJEmSJPXIoC1JkiRJUo8M2pIkSZIk9cigLUmSJElSjwzakiRJkiT1yKAtSZIkSVKPDNqSJEmSJPXIoC1JkiRJUo8M2pIkSZIk9cigLUmSJElSjwzakiRJkiT1yKAtSZIkSVKPDNqSJEmSJPXIoC1JkiRJUo8M2pIkSZIk9cigLUmSJElSjwzakiRJkiT1yKA9iSR/Xkb97JbkQcuor1OTbLEs+hrqd3aSlw283iLJAUuj7RHbZ+SYJUmSJN0zGbSXD7sByyRoL01JZk2yeTZwWxiuqvOr6s09dX2HtiVJkiRpJhm0pyGd/ZNckmRBkl1a+VpJTkpyQSt/QSufneTSJIck+WGS7yRZfUzbOwFbAF9MclGS1ZM8LcmFrc1Dk6w6Zt9nJTly4PV2SY5pzw9Mcn7rf98x+/954PlOSQ5rz9dL8rUk57WfrSc5N/sk+XySs4DPt2M/o52TC5I8qVWdB2zbjvFtQ2O9X5JvJJmf5PtJNp2kv6e0Ni5q5+jeI9pePcmX23twFDDu3O/RztH5i264dlyXkiRJkrRYDNrT8yJgDvBYYAdg/yTrAzcBO1bVZsD2wH8mSdtnI+CTVfVo4BrgxaMarqqvAucDL6+qOUABhwG7VNUmwCxgzzHjOhF4fJI12+tdgC+353tX1RbApsBTJguvI3wM+GhVbdnG/Zkp6j8K2KGqXgr8Bnh6Oye7ABPLw/cCzqiqOVX10aH99wUurKpNgfcBR0zS1zuBN7RztS1w44i29wRuqKpHAu8HNh/VUFUdXFVbVNUWK6+x9hSHKEmSJEnTY9Cenm2AL1XVoqq6GjgN2BII8G9J5tOF3g2AB7Z9rqyqi9rzH9Atb56OR7R9L2+vDweePKpiVd0CHA88ry3bngt8s21+SZILgAuBR9OF4enaAfhEkouAo4H7JFlrkvpHV9WN7fkqwCFJFgBHTrPfbYDPt2M6GVg3yX3G1D0L+EiSNwPrtHMw7MnAF1p784H50xiDJEmSJPVismtqNbWXA+sBm1fVX5MsBFZr224eqLeIMcuXe/Bl4I3AH4Dzq+q6JA+jm/ndsqr+2JaErzZi3xp4Prh9JeAJVXXTNMdw/cDztwFX083+r0Q369+bqpqX5FjgOcBZSZ7ZZ/uSJEmStKSc0Z6eM4BdkqycZD26GdNzgbWB37SQvT3w0LvY/nXAvdvzHwOzkzy8vX4F3Qz6OKcBmwGv4fZl4/ehC7/XJnkg8Owx+16d5JFJVgJ2HCj/DvCmiRdJ5izGsawN/Kqqbm1jX7mVDx7jsDPoPrQgyXbA76rqT6MqJtmwqhZU1YeB84CNR7R9Ou3maEkeQ7d8XpIkSZKWCYP29BxFt/z4YuBk4N1V9Wvgi8AWbZn0rsBld7H9w4CD2lLtALsDR7Z2bwUOGrdjVS0CjqEL08e0sovploxfBvw33XLrUfZq+5wN/Gqg/M3tuOYn+RHwusU4lk8Br0xyMV0Inpjtng8sSnJxkrcN7bMPsHlbgj8PeOUk7b+13ZRuPvBX4LgRbR8IrJXkUuADdEv3JUmSJGmZSFVNXUu6m9tz7/3quEVOfEuSJEnLg4Xz5s70EEbJ1FU6zmhLkiRJktQjb4a2DCX5JDD8ndQfq6rPTWPfo4CHDRW/p6pO6Gt8U/S/O/CWoeKzquoNd4f+JEmSJKkvBu1laElCYlXtOHWtpad9GDDlBwIran+SJEmS1BeXjkuSJEmS1CODtiRJkiRJPTJoS5IkSZLUI4O2JEmSJEk9MmhLkiRJktQjg7YkSZIkST0yaEuSJEmS1CODtiRJkiRJPTJoS5IkSZLUo1kzPQBpebDJBmtz4OvnzvQwJEmSJN0NOKMtSZIkSVKPDNqSJEmSJPXIoC1JkiRJUo8M2pIkSZIk9cigLUmSJElSjwzakiRJkiT1yKAtSZIkSVKPDNqSJEmSJPXIoC1JkiRJUo9mzfQApOXBgl9ey+y9jp3pYUiStFQtnDd3pocgSfcIzmhLkiRJktQjg7YkSZIkST0yaEuSJEmS1CODtiRJkiRJPTJoS5IkSZLUI4O2JEmSJEk9MmhLkiRJktQjg7YkSZIkST0yaEuSJEmS1CODtiRJkiRJPTJoS5IkSZLUI4O2JEmSJEk9MmhLkiRJktQjg7YkSZIkST0yaEuSJEmS1CODtiRJkiRJPTJoS5IkSZLUI4P2PVCS2Uletoz62i3JJ5ZBP29NssZMjkGSJEmSwKC93Eqy8lJqdxYwG1gmQXsZeiswMmhLkiRJ0rJk0J4BbUb5siRfTHJpkq8mWSPJwiQfTnIBsHOSOUm+n2R+kqOS3Lftf2qSjyW5KMklSbZq5WsmOTTJuUkuTPKCVr5bkqOTnAycBMwDtm37vy3J6UnmDIzvzCSPHTHuldoY1xko+0mSByZ5XpJzWr8nJnngiP0PS7LTwOs/Dzx/V5Lz2rHuO8m5WzPJsUkubse+S5I3Aw8CTklySqu3e5LLk5wLbD2mrT2SnJ/k/EU3XDuuS0mSJElaLAbtmfMI4FNV9UjgT8DrW/nvq2qzqvoycATwnqraFFgAvH9g/zWqak7b79BWtjdwclVtBWwP7J9kzbZtM2CnqnoKsBdwRlXNqaqPAp8FdgNI8vfAalV18fCAq+pW4JvAjq3u44GfVdXVwJnAE6rqccCXgXdP90QkeQawEbAVMAfYPMmTx1R/FnBVVT22qh4DHF9VBwBXAdtX1fZJ1gf2pQvY2wCPGtVQVR1cVVtU1RYrr7H2dIcrSZIkSZMyaM+c/6uqs9rzL9AFQoCvACRZG1inqk5r5YcDg+HzSwBVdTpwnzbL/AxgryQXAacCqwEPafW/W1V/GDOWI4HnJlkFeBVw2CTj/gqwS3v+DxPjBR4MnJBkAfAu4NGTtDHsGe3nQuACYGO64D3KAuDpbeZ/26oaNRX9eODUqvptVf1lYIySJEmStNQZtGdOjXl9/RLsH+DFbaZ6TlU9pKounardqroB+C7wAuAlwBcn6fd7wMOTrAe8EPh6K/848Imq2gR4LV3IH3YL7XcuyUrAvVp5gP0Gxv3wqvrsmLFeTjc7vwD4YJJ/nWSskiRJkrTMGbRnzkOSPLE9fxnd0uvbtJnaPybZthW9AjhtoMouAEm2Aa5t9U8A3pQkbdvjxvR9HXDvobLPAAcA51XVH8cNuqoKOAr4CHBpVf2+bVob+GV7/soxuy8ENm/Pnw+s0p6fALwqyVpt3BskecCoBpI8CLihqr4A7E8XuoeP6RzgKUnWbbP0O487HkmSJEnq26yZHsA92I+BNyQ5FPgRcCDwpqE6rwQOal9bdQWw+8C2m5JcSBdWX9XK/h/wX8D8NmN8JfDcEX3PBxYluRg4rKo+WlU/SPIn4HPTGPtXgPNo13U3+wBHJvkjcDLwsBH7HQJ8s/V7PG2Wvaq+k+SRwPfaZwR/Bv4R+M2INjahu/b8VuCvwJ6t/GDg+CRXteu096Gbfb8GuGgaxyRJkiRJvUg3QallKcls4Jh2M6+7sv+pwDur6vwex/Qguuu6N243PbtH2XPv/eq4RZvO9DAkSVqqFs6bO9NDkKQVWaZb0aXjIsmudMut974nhmxJkiRJ6pNLx2dAVS0E7tJsdtt/u94G07V3BN1Xid0mye7AW4aqnlVVb+iz73GSrEv3nd/DnjZwXbgkSZIkLXcM2hqpqj7H9K7XXlr9/57uO7UlSZIkaYXi0nFJkiRJknpk0JYkSZIkqUcGbUmSJEmSemTQliRJkiSpRwZtSZIkSZJ6ZNCWJEmSJKlHBm1JkiRJknpk0JYkSZIkqUcGbUmSJEmSemTQliRJkiSpR7NmegDS8mCTDdbmwNfPnelhSJIkSbobcEZbkiRJkqQeGbQlSZIkSeqRQVuSJEmSpB4ZtCVJkiRJ6pFBW5IkSZKkHhm0JUmSJEnqkUFbkiRJkqQeGbQlSZIkSeqRQVuSJEmSpB7NmukBSMuDBb+8ltl7HTvTw5C0hBbOmzvTQ5AkSXJGW5IkSZKkPhm0JUmSJEnqkUFbkiRJkqQeGbQlSZIkSeqRQVuSJEmSpB4ZtCVJkiRJ6pFBW5IkSZKkHhm0JUmSJEnqkUFbkiRJkqQeGbQlSZIkSeqRQVuSJEmSpB4ZtCVJkiRJ6pFBW5IkSZKkHhm0JUmSJEnqkUFbkiRJkqQeGbQlSZIkSerRPT5oJ9knyTsn2X5Ykp1GlM9J8pwp2t4uyZPu4rhel2TXu7Lv8ijJW5OsMfD620nWWRptD23bLckn+uhHkiRJkqbjHh+0l8AcYNKgDWwH3KWgXVUHVdURd2XfmZDOZL9PbwVuC8NV9Zyquqan7u/QtiRJkiTNpHtk0E6yd5LLk5wJPKKVbZjk+CQ/SHJGko0Hdtkhyfltn+cmuRfwAWCXJBcl2WVEH7OB1wFva3W2TTI7yclJ5ic5KclDJhnjbTPtSU5N8uEk57YxbNvKV07yH0kuaW2+qZU/LcmFSRYkOTTJqq18YZL92njOT7JZkhOS/G+S1w30/a4k57U2951kjLOT/DjJEcAlwN8mObC1/cOJfZO8GXgQcEqSUwbGcv/2/O3tGC5J8tZJ+lszybFJLm51dxnT9u7tPJ0LbD1Je3u0sZ6/6IZrx1WTJEmSpMVyjwvaSTYH/oHbZ6S3bJsOBt5UVZsD7wQ+NbDbbGArYC5wEN15+1fgK1U1p6q+MtxPVS1sdT/a6pwBfBw4vKo2Bb4IHLAYQ59VVVvRzd6+v5Xt0cY2Z6LNJKsBhwG7VNUmwCxgz4F2fl5Vc4AzWr2dgCcAE6H4GcBG7XjnAJsnefIk49oI+FRVPbqqfgbsXVVbAJsCT0myaVUdAFwFbF9V2w/u3N6P3YHHt3G8JsnjxvT1LOCqqnpsVT0GOH647STrt2PZGtgGeNS4gVfVwVW1RVVtsfIaa09yiJIkSZI0ffe4oA1sCxxVVTdU1Z+Ao4HV6JZ4H5nkIuDTwPoD+/xPVd1aVT8BrgA2Hm50mp4I/Hd7/nm6IDhdX2+PP6AL1wA7AJ+uqlsAquoPdDP0V1bV5a3O4cBgUD66PS4Azqmq66rqt8DN7ZrpZ7SfC4EL6I51o0nG9bOq+v7A65ckuaDt/2gmCbrNNnTvx/VV9ed2nNuOqbsAeHqb3d+2qkZNQz8eOLWqflsF4I/4AAAgAElEQVRVfwHu9CGIJEmSJC1Ns2Z6AMuJlYBr2kzvKDXF62Xh5va4iCV73ybauXXg+cTrWUCA/arq09Ns7/qJJ0keRrcaYMuq+mOSw+g+xOhFVV2eZDO6lQgfTHJSVX2gr/YlSZIkqQ/3xBnt04EXJlk9yb2B5wE3AFcm2Rluu7HXYwf22TnJSkk2BP4O+DFwHXDvKfoarnM23bJ1gJfTLd9eEt8FXptkVhv3/drYZid5eKvzCuC0xWjzBOBVSdZqbW6Q5AHT3Pc+dMH72iQPBJ49sG3c+TqD7v1YI8mawI6MOS9JHgTcUFVfAPYHNhvR9jl0S9bXTbIKsPM0xy5JkiRJvbjHBe2quoBuOfHFwHHAeW3Ty4FXJ7kY+CHwgoHdfg6c2+q/rqpuAk4BHjXuZmjNt4AdJ26GBrwJ2D3JfLoA/JYlPJzPtLHNb+N+WRvb7nTL4BfQzVQfNN0Gq+o7dMvbv9f2/ypTf6Awse/FdEvGL2ttnDWw+WDg+Ikblg3scwHdteLn0oXkz1TVhWO62AQ4ty3vfz/wweG2q+pXwD7A91r/l05n7JIkSZLUl1TNxCpoafmy59771XGLNp3pYUhaQgvnzZ3pIUiSpLuvTLfiPW5GW5IkSZKkpcmbofUgye7ceRn4WVX1hmnsuzd3vo74yKr6UF/jW1JJ1gVOGrHpaVX1+xW9P0mSJEnqk0G7B1X1OeBzd3HfDwHLTagepYXbcXdkX+H7kyRJkqQ+uXRckiRJkqQeGbQlSZIkSeqRQVuSJEmSpB4ZtCVJkiRJ6pFBW5IkSZKkHhm0JUmSJEnqkUFbkiRJkqQeGbQlSZIkSeqRQVuSJEmSpB4ZtCVJkiRJ6tGsmR6AtDzYZIO1OfD1c2d6GJIkSZLuBpzRliRJkiSpRwZtSZIkSZJ6ZNCWJEmSJKlHBm1JkiRJknpk0JYkSZIkqUcGbUmSJEmSemTQliRJkiSpRwZtSZIkSZJ6ZNCWJEmSJKlHs2Z6ANLyYMEvr2X2XsfO9DC0DCycN3emhyBJkqS7OWe0JUmSJEnqkUFbkiRJkqQeGbQlSZIkSeqRQVuSJEmSpB4ZtCVJkiRJ6pFBW5IkSZKkHhm0JUmSJEnqkUFbkiRJkqQeGbQlSZIkSeqRQVuSJEmSpB4ZtCVJkiRJ6pFBW5IkSZKkHhm0JUmSJEnqkUFbkiRJkqQeGbQlSZIkSeqRQVuSJEmSpB4ZtCVJkiRJ6pFBuydJzp5GnbcmWWNZjGegzxcmedSy7HOo/ynPy1D9w5LstJj7jD3GJLOTXLI47UmSJEnSkjBo96SqnjSNam8FRgbtJCvf1b6n2PeFwDIP2klmwbTPy5KakWOUJEmSpFEM2j1J8uf2uF2SU5N8NcllSb6YzpuBBwGnJDllYp8k/5nkYuCJSZ6W5MIkC5IcmmTVSfpbmOTDSS4Adk7ymiTnJbk4ydeSrJHkScDzgf2TXJRkw/ZzfJIfJDkjycaT9HFYkoOSnJ/k8iTPbeUrJ9m/9Tc/yWsHjv2MJEcDPxo6L2n7XNKOb5eB8k8k+XGSE4EHTHGe5yX5Uev3P8Yc4+btPFwMvGGStvZox3b+ohuunaxbSZIkSZq2WTM9gLupxwGPBq4CzgK2rqoDkrwd2L6qftfqrQmcU1XvSLIa8BPgaVV1eZIjgD2B/5qkn99X1WYASdatqkPa8w8Cr66qj7fQe0xVfbVtOwl4XVX9JMnjgU8BT52kj9nAVsCGdB8SPBzYFbi2qrZsHwacleQ7rf5mwGOq6sqhdl4EzAEeC9wfOC/J6cATgUfQzUg/kC6gHzpqIEnWBXYENq6qSrJOVV0z4hjnA2+sqtOT7D/uwKrqYOBggD333q9YNMlZkCRJkqRpckZ76Ti3qn5RVbcCF9GF1VEWAV9rzx8BXFlVl7fXhwNPnqKfrww8f0ybTV4AvJwu6N9BkrWAJwFHJrkI+DSw/hR9/E9V3VpVPwGuADYGngHs2to4B1gX2KjVP3dEyAbYBvhSVS2qqquB04At2zFOlF8FnDzJWK4FbgI+m+RFwA0jjnEdYJ2qOr0VfX6K45MkSZKkXjmjvXTcPPB8EePP801VtSTzqNcPPD8MeGFVXZxkN2C7EfVXAq6pqjmL0UeNeB3gTVV1wuCGJNsNjalXVXVLkq2ApwE7AW9k8tl4SZIkSVrmnNFetq4D7j1m24+B2W1pNsAr6GZ9p+vewK+SrEI3o32nPqvqT8CVSXaG266PfuwU7e6cZKUkGwJ/18Z5ArBn64skf59kzSnaOQPYpV3fvR7dTPa5wOkD5esD249roM3Ir11V3wbeRrcMffgYrwGuSbJN2/byOzUkSZIkSUuRQXvZOhg4fuJmaIOq6iZgd7pl3QuAW4GDFqPtf6Fbxn0WcNlA+ZeBd7WbrG1IFzxf3W4U9kPgBVO0+3O6QHwc3bXdNwGfobuW+oJ0X531aaZeHXEUMB+4mG55+Lur6tet/CetvSOA703Sxr2BY9o12GcCbx9zjLsDn2xL2zPFuCRJkiSpV6kaXhksdZIcxsBNxu7O9tx7vzpu0aYzPQwtAwvnzZ3pIUiSJGnFNO1JPGe0JUmSJEnqkTdDW84lOQp42FDxe4ZvRLaEfewN7DxUfGRV7dZXH4trWRy3JEmSJC0NBu3lXFXtuAz6+BDwoaXdz+JYFsctSZIkSUuDS8clSZIkSeqRQVuSJEmSpB4ZtCVJkiRJ6pFBW5IkSZKkHhm0JUmSJEnqkUFbkiRJkqQeGbQlSZIkSeqRQVuSJEmSpB4ZtCVJkiRJ6pFBW5IkSZKkHs2a6QFIy4NNNlibA18/d6aHIUmSJOluwBltSZIkSZJ6ZNCWJEmSJKlHBm1JkiRJknpk0JYkSZIkqUcGbUmSJEmSemTQliRJkiSpRwZtSZIkSZJ6ZNCWJEmSJKlHBm1JkiRJkno0a6YHIC0PFvzyWmbvdexMD0PTsHDe3JkegiRJkjQpZ7QlSZIkSeqRQVuSJEmSpB4ZtCVJkiRJ6pFBW5IkSZKkHhm0JUmSJEnqkUFbkiRJkqQeGbQlSZIkSeqRQVuSJEmSpB4ZtCVJkiRJ6pFBW5IkSZKkHhm0JUmSJEnqkUFbkiRJkqQeGbQlSZIkSeqRQVuSJEmSpB4ZtCVJkiRJ6pFBW5IkSZKkHhm0l2NJ/jzTY1iWksxO8rK7sN86SV4/yfbDkuy0ZKOTJEmSpOkxaGt5MhtY7KANrAOMDdqSJEmStCwZtFcA6eyf5JIkC5Ls0srXSnJSkgta+Qta+ewklyY5JMkPk3wnyeqTtP+aJOcluTjJ15Ks0coPS3Jgku8nuSLJdkkObW0fNrD/S1v/lyT58ED5nwee7zSxT2v3gCRnt3YnZpvnAdsmuSjJ28aM9dFJzm115ifZqO23YSvbv52vTyT5cZITgQeMaWuPJOcnOX/RDddO452QJEmSpKkZtFcMLwLmAI8FdgD2T7I+cBOwY1VtBmwP/GeStH02Aj5ZVY8GrgFePEn7X6+qLavqscClwKsHtt0XeCLwNuBo4KPAo4FNksxJ8iDgw8BT2xi3TPLCaRzT+sA2wHPpgjLAXsAZVTWnqj46Zr/XAR+rqjnAFsAv2n7/2/Z7F7Aj8AjgUcCuwJNGNVRVB1fVFlW1xcprrD2NIUuSJEnS1AzaK4ZtgC9V1aKquho4DdgSCPBvSeYDJwIbAA9s+1xZVRe15z+gW5Y9zmOSnJFkAfByuiA94VtVVcAC4OqqWlBVtwI/bG1uCZxaVb+tqluALwJPnsYxfaOqbq2qHw2MeTq+B7wvyXuAh1bVjSPqPJnbz9dVwMmL0b4kSZIkLRGD9ort5cB6wOZthvdqYLW27eaBeouAWZO0cxjwxqraBNh3oI3Bdm4davPWKdoEqIHnqw1tG2wrTFNV/TfwfOBG4NtJnjrdfSVJkiRpWTBorxjOAHZJsnKS9ehmbM8F1gZ+U1V/TbI98NC72P69gV8lWYUuvC+Oc4GnJLl/kpWBl9LNuANcneSRSVaiW849levaWMZK8nfAFVV1APBNYNMR+53O7edrfbpl9ZIkSZK0TEw1I6nlw1F010lfTDdL/O6q+nWSLwLfaku+zwcuu4vt/wtwDvDb9jhp2B1UVb9KshdwCt3M9LFV9c22eS/gmNbu+cBaUzQ3H1iU5GLgsDHXab8EeEWSvwK/Bv6tqv6Q5KwklwDHAe+mu2b8R8DP6ZabS5IkSdIyke7yW+mebc+996vjFm0608PQNCycN3emhyBJkqR7pmlf8urScUmSJEmSeuTS8XuQJJ8Eth4q/lhVfW4mxjOZJM+k+9qwQVdW1XSu9ZYkSZKkGWPQvgepqjfM9Bimq6pOAE6Y6XFIkiRJ0uJy6bgkSZIkST0yaEuSJEmS1CODtiRJkiRJPTJoS5IkSZLUI4O2JEmSJEk9MmhLkiRJktQjg7YkSZIkST0yaEuSJEmS1CODtiRJkiRJPTJoS5IkSZLUo1kzPQBpebDJBmtz4OvnzvQwJEmSJN0NOKMtSZIkSVKPDNqSJEmSJPXIoC1JkiRJUo8M2pIkSZIk9cigLUmSJElSjwzakiRJkiT1yKAtSZIkSVKPDNqSJEmSJPXIoC1JkiRJUo9mzfQApOXBgl9ey+y9jp2RvhfOmzsj/UqSJElaOpzRliRJkiSpRwZtSZIkSZJ6ZNCWJEmSJKlHBm1JkiRJknpk0JYkSZIkqUcGbUmSJEmSemTQliRJkiSpRwZtSZIkSZJ6ZNCWJEmSJKlHBm1JkiRJknpk0JYkSZIkqUcGbUmSJEmSemTQliRJkiSpRwZtSZIkSZJ6ZNCWJEmSJKlHBm1JkiRJknpk0JYkSZIkqUcG7Z4k+UCSHaaos12SJy2rMbU+Zyd52bLsc6j/Kc/LUP3tkhyzmH1MeoxJTk2yxeK0KUmSJEl3lUG7J1X1r1V14hTVtgNGBu0ks+5q31PsOxuYkaCdZOVpnpclNZsZOkZJkiRJGmbQ5rYZ0UuTHJLkh0m+k2T1JHOSfD/J/CRHJbnvJG0clmSn9nxhkn2TXJBkQZKNk8wGXge8LclFSbZt+xyU5Bzg35PcL8k3Wn/fT7LpJP3tk+TzSc4CPt+O4YzW5wUDM+fzgG1bn29LsnKS/ZOc1/p57SR9bJfk9CTHJvlxG+tKbdszknyv9XVkkrUGjv3DSS4Adh46L09LcmE7J4cmWbWVPyvJZW2fF03xXj2lHctFra17jzjG1ZN8ub2nRwGrj2lrjyTnJzl/0Q3XTtatJEmSJE2bQft2GwGfrKpHA9cALwaOAN5TVZsCC4D3L0Z7v6uqzYADgXdW1ULgIOCjVTWnqs5o9R4MPKmq3g7sC1zY+ntf638yjwJ2qKqXAr8Bnt763AU4oNXZCzij9flR4NXAtVW1JbAl8JokD5ukj62AN7W+NgRelOT+wD+3vjcDzgfePrDP76tqs6r68kRBktWAw4BdqmoTYBawZys/BHgesDnwN1Mc8zuBN1TVHGBb4MYRx7gncENVPZLuPdt8VENVdXBVbVFVW6y8xtpTdCtJkiRJ02PQvt2VVXVRe/4DulC5TlWd1soOB568GO19faCt2ZPUO7KqFrXn2wCfB6iqk4F1k9xnkn2Prqob2/NVgEOSLACOpAvGozwD2DXJRcA5wLp0HzKMc25VXdHG+KU2xie09s9q7bwSeOjAPl8Z0c4j6M7x5e31xPncuJX/pKoK+MIkYwE4C/hIkjfTvT+3jKjz5Il2qmo+MH+KNiVJkiSpN3f5uuC7oZsHni8C1umpvUVMfp6vX4I+Bvd9G3A18Fi6D1BuGrNPgDdV1QnT7KNGvA7w3TaTPtW4elVV85IcCzyHLug/c2n1JUmSJEl3hTPa410L/DHJtu31K4DTJqk/HdcB955k+xnAy6G7Pppu+fmfptn22sCvqupWurGuPKbPE+iWbK/S+vn7JGtO0u5WSR7Wrs3eBTgT+D6wdZKHtzbWTPL3U4zvx8DsiX24/Xxe1so3bOXjwjutrw2rakFVfRg4j25GfPgYT6fdHC3JY4Cx17pLkiRJUt+c0Z7cK4GDkqwBXAHsvoTtfQv4apIX0F33PGwf4NAk84EbWv/T9Snga0l2BY7n9lnl+cCiJBfTXSP9Mbql7BckCfBb4IWTtHse8Ang4cApwFFVdWuS3YAvTdzQjO6a7ctHNwFVdVOS3YEj213SzwMOqqqbk+wBHJvkBroPGyb7MOKtSbYHbgV+CBzXng8e44HA55JcClxKt3xfkiRJkpaJdJfFSnfWZtXfWVXPnemxLG177r1fHbdoZia+F86bOyP9SpIkSVosmW5Fl45LkiRJktQjl44vpiSfBLYeKv5YVX1uKfW3O/CWoeKzquoNPfaxCe1u5wNurqrHA6f21c/iWBbHLUmSJElLg0F7MS3roNcC/FIJ8QN9LADmLM0+FteyOG5JkiRJWhpcOi5JkiRJUo8M2pIkSZIk9cigLUmSJElSjwzakiRJkiT1yKAtSZIkSVKPDNqSJEmSJPXIoC1JkiRJUo8M2pIkSZIk9cigLUmSJElSj2bN9ACk5cEmG6zNga+fO9PDkCRJknQ34Iy2JEmSJEk9MmhLkiRJktQjg7YkSZIkST0yaEuSJEmS1CODtiRJkiRJPTJoS5IkSZLUI4O2JEmSJEk9MmhLkiRJktQjg7YkSZIkST2aNdMDkJYHC355LbP3OnbktoXz5i7j0UiSJElakTmjLUmSJElSjwzakiRJkiT1yKAtSZIkSVKPDNqSJEmSJPXIoC1JkiRJUo8M2pIkSZIk9cigLUmSJElSjwzakiRJkiT1yKAtSZIkSVKPDNqSJEmSJPXIoC1JkiRJUo8M2pIkSZIk9cigLUmSJElSjwzakiRJkiT1yKAtSZIkSVKPDNqSJEmSJPXIoK3eJdk/yQ+T7L8U2v5Mkkf13a4kSZIk9WXWTA9Ad0t7APerqkXTqZxkVlXdMp26VfVPSzQySZIkSVrKnNHWHSTZNcn8JBcn+XyS2UlObmUnJXlIq3dYkgOSnJ3kiiQ7tfKjgbWAHyTZZYr9D0pyDvz/9u482rKyvBPw7xVUWiEYpeOyHVKKqC0ChZQ4oji0A6QhGlEcWkWXBDW0xNYlNllOkQa1E1uMqLQDqJhGWMaQKBBbUQijIEOBihNgMKYFBwJCEOHtP84uvKu899a5VZt761Y9z1p33X32/vbe79nnW1X1O9+3d+U9VfX2qjquqs6sqqur6nlV9Z6qWl1Vp1bVXYf9vlpVq4blG6vq8KHWc6vqvsP67YfXq6vqXVV14xJcSgAAYDMlaHOHqtoxyZ8leVp375Lk9Uk+kOS47t45yfFJjpqxy/2SPCnJHyQ5Mkm6e58kN3f3yu4+YR37PyDJE7r7DcPr7ZM8Lck+ST6d5PTu3inJzUn2nqXkeyY5d6j1jCSvHta/P8n7h32vmef9HlhVF1TVBbfddP26LxAAAMAUBG1melqSE7v7uiTp7p8leXySzwzbP5VJsF7j8919e3d/M8l95zjmfPufuNb08lO6+9Ykq5NskeTUYf3qJCtmOfavkvz9sHzhjDaPT3LisPyZzKG7j+nuVd29aot7bDtXMwAAgAURtNkQt8xYrvXY/5ezHa+7b09ya3f3sP72zP48gZltbpujDQAAwKIStJnpK0n2q6r7JElV3TvJ2Un2H7a/JMmZCzzmhu6/Ps5N8kfD8v7zNQQAABibEUDu0N2XV9XhSb5WVbcluSjJwUk+UVVvSnJtkgMWeNgN3X99HJLk01V1WCbTz92ADQAALJr6zcxb2DRU1T0yeSBbV9X+SV7U3fvOt89rDjuiT7lt51m3XXXkbM9hAwAANjNT3y5rRJtN0W5J/qqqKskvkrxyiesBAAA2I4I2m5zuPjPJLktdBwAAsHnyMDQAAAAYkaANAAAAIxK0AQAAYESCNgAAAIxI0AYAAIARCdoAAAAwIkEbAAAARiRoAwAAwIgEbQAAABiRoA0AAAAjErQBAABgRII2AAAAjGjLpS4ANgY73X/bfOi1ey91GQAAwCbAiDYAAACMSNAGAACAEQnaAAAAMCJBGwAAAEYkaAMAAMCIBG0AAAAYkaANAAAAIxK0AQAAYESCNgAAAIxI0IYkq390fVYc+oWlLgMAANgECNoAAAAwIkEbAAAARiRoAwAAwIgEbQAAABiRoA0AAAAjErQBAABgRII2AAAAjEjQBgAAgBEJ2gAAADAiQRsAAABGJGgDAADAiARtAAAAGJGgDQAAACMStAEAAGBEgjYAAACMSNAGAACAEQna3KGqzl7qGtZHVa2sqr3m2X5VVW23mDUBAACbL0GbO3T3E5a6hvW0MsmcQRsAAGAxCdrcoapuHH7vWVVfraqTqurbVXV8VdWw7TFVdXZVXVJV51fVNlW1VVV9oqpWV9VFVfXUoe0rqurzVfWlYVT5T6rqDUObc6vq3kO77avq1Kq6sKrOrKpHzFPjflV12XD+M6rqbknemeSFVXVxVb2wqu5TVf9QVZdX1UeT1J1+8QAAAAaCNnPZNckhSR6Z5CFJnjiE2hOSvL67d0nyjCQ3J3ldku7unZK8KMlxVbXVcJxHJXleksckOTzJTd29a5JzkrxsaHNMkoO7e7ckb0xy9Dx1vTXJs4bz79PdvxrWndDdK7v7hCRvS/KP3b1jkr9J8qDZDlRVB1bVBVV1wW03Xb/Q6wMAADArQZu5nN/d13T37UkuTrIiycOT/Li7v54k3f2v3f3rJE9K8ulh3beTXJ3kYcNxTu/uG7r72iTXJ/m7Yf3qJCuqauskT0hyYlVdnOQjSe43T11nJTm2ql6dZIs52jx5Rj1fSPLz2Rp19zHdvaq7V21xj23nvxoAAABT2nKpC2CjdcuM5duy/n1l5nFun/H69uGYd0nyi+5eOc3Buvugqnpskr2TXFhVu61nXQAAAHcKI9osxBVJ7ldVj0mS4f7sLZOcmeQlw7qHZTJV+4ppDtjd/5rkyqrab9i/qmqXudpX1fbdfV53vzXJtUkemOSGJNvMaHZGkhcP7Z+T5HcX9C4BAAA2gKDN1Ib7oV+Y5ANVdUmSLyXZKpN7qu9SVaszuYf7Fd19y9xH+i0vSfKq4ZiXJ9l3nrbvHR66dlmSs5NckuT0JI9c8zC0JO9I8uSqujyT+8N/uKA3CgAAsAGqu5e6BlhyrznsiD7ltp1z1ZF7L3UpAADAxmnq/83IiDYAAACMyMPQ2ChV1WFJ9ltr9YndffhS1AMAADAtQZuN0hCohWoAAGDZMXUcAAAARiRoAwAAwIgEbQAAABiRoA0AAAAjErQBAABgRII2AAAAjEjQBgAAgBEJ2gAAADAiQRsAAABGJGgDAADAiARtSLLT/bfNVUfuvdRlAAAAmwBBGwAAAEYkaAMAAMCIBG0AAAAYkaANAAAAIxK0AQAAYESCNgAAAIxI0AYAAIARCdoAAAAwIkEbAAAARiRoQ5LVP7p+qUsAAAA2EYI2AAAAjEjQBgAAgBEJ2gAAADAiQRsAAABGJGgDAADAiARtAAAAGJGgDQAAACMStAEAAGBEgjYAAACMSNAGAACAEQnaAAAAMCJBGwAAAEYkaAMAAMCIBG0AAAAYkaANAAAAIxK0AQAAYESCNgAAAIxowUG7qt5eVW+cZ/uxVfX8WdavrKq91nHsPavqCQutadj3oKp62frsuzGqqkOq6h4zXn+xqu51J51rn6o6dD32O3uk86+oqpur6uIZP3ebp/2eVfX3Y5wbAABgbFsu4rlWJlmV5IvztNkzyY1JFhzguvvD61fW0qiqSlLdffscTQ5J8ukkNyVJd8/7JcWG6O6Tk5y8Hvut15cic/h+d68c8XgAAABLYqoR7ao6rKq+U1X/mOThw7rtq+rUqrqwqs6sqkfM2OUZVXXBsM8fDKOT70zywmG08oWznGNFkoOS/OnQZo9hpPMrVXVpVX25qh40T413jLRX1Ver6t1Vdf5Qwx7D+i2q6n9W1WXDMQ8e1j+9qi6qqtVV9fGquvuw/qqqOmKo54KqenRVnVZV36+qg2ac+01V9fXhmO+Yp8YVVXVFVX0yyWVJHlhVHxqOffmafavqvyb5D0lOr6rTZ9Sy3bD8huE9XFZVh6zjfN8eZhl8p6qOr6pnVNVZVfXdqtp9aPeKqvqrYXm/4biXVNUZw7odh2t58fAedxjW3zj83nO45icN5zt++CIhVbXXsO7CqjpqISPRVbV7VZ0zfDZnV9XDZ2nzlBmj4BdV1TbTfiZVdeBw7S+47abrpy0LAABgXusM2lW1W5L9MxmR3ivJY4ZNxyQ5uLt3S/LGJEfP2G1Fkt2T7J3kw8N53prkhO5e2d0nrH2e7r5qaPu+oc2ZST6Q5Lju3jnJ8UmOWsB727K7d89kZPhtw7oDh9pWrjlmVW2V5NgkL+zunTIZ5X/NjOP8cBhpPXNo9/wkj0uyJhQ/M8kOw/tdmWS3qnryPHXtkOTo7t6xu69Oclh3r0qyc5KnVNXO3X1Ukn9O8tTufurMnYfP44Akjx3qeHVV7TrP+R6a5C+SPGL4eXGSJ2Xymf33Wdq/NcmzunuXJPsM6w5K8v7hOqxKcs0s++2aybV+ZJKHJHnicG0/kuQ5Qz/59/PUuf2MwPzBYd23k+zR3bsOdf2PWfZ7Y5LXDbXtkeTmaT+T7j6mu1d196ot7rHtPKUBAABMb5qp43sk+ZvuvilJqurkJFsleUKSE4eByyS5+4x9PjtMif5uVf0gk4C3Ph6f5HnD8qeSvGcB+35u+H1hJuE6SZ6R5MPd/esk6e6fVdUuSa7s7u8MbY5L8rok/2t4vWZK9eokW3f3DUluqKpbanLP9DOHn4uGdltnEvLOmKOuq7v73BmvX1BVB2byWdwvk6B66Tzv60mZfB6/TJKq+lwmn9FFc7S/srtXD20vT/Ll7u6qWj3jusx0VpJjq+qz+c01PCfJYVX1gCSf6+7vzrLf+d19zd0gf6oAAAyJSURBVHCei4dj35jkB9195dDmrzP5smM2s00d3zbJccMIeie56xz1/mVVHT/Uds0QtBfymQAAAIxmfe/RvkuSX8xzT22v4/ViuGX4fVs27F70Nce5fcbymtdbJqkkR3T3R6Y83i/XLFTVgzMZkX1Md/+8qo7N5EuMMa1d88z381vXpbsPqqrHZjIb4cKq2q27P1NV5w3rvlhVf9zdX5nnPBt6zdf48ySnd/dza3JrwVdnqffIqvpCJrMtzqqqZ2XhnwkAAMBoprlH+4wkf1hV/264//U/Z/KAriurar9k8mCvYWR4jf2q6i5VtX0m04ivSHJDkm3Wca6125ydybT1JHlJJtO3N8SXkvxxVW051H3vobYVVfXQoc1/SfK1BRzztCSvrKqth2Pev6p+b8p9fyeT4H19Vd03yXNmbJvrep2Zyedxj6q6Z5LnZsOvyx2qavvuPq+735rk2kzuI39IJiPTRyX520ymuU/jiiQPGUJykvzWvfnrsG2SHw3Lr5in3tXd/e4kX89k9sSGfCYAAAAbZJ1Bu7u/keSEJJckOSWTMJNMgu+rquqSJJcn2XfGbj9Mcv7Q/qDu/rckpyd5ZM3xMLTB3yV57tBmjyQHJzmgqi7NJAC/fqFvcC0fHWq7dKj7xUNtB2QyDX51JiO9Uz/BvLv/Iclnkpwz7H9S1v2Fwpp9L8lkevO3h2OcNWPzMUlOreFhaDP2+UYm94qfn+S8JB/t7rmmja+P99bkoXCXZfJFxyVJXpDksmFK+KOSfHKaA3X3zUlem8n7uDCTLw+uT5KqWlVVH13HId6T5Iiquihzj5AfMjy87dIktyY5ZUM+EwAAgA1V3Usxq5vNRVVt3d03Dk8h/2CS73b3+5a6rrW95rAj+kOHv2WpywAAADZete4mE1P9916wAV49jIRfnslUcPdNAwAAm7QxHli1YFV1QH57GvhZ3f26KfY9LMl+a60+sbsPH6u+DVVV90ny5Vk2Pb27f7rcz7cQw+j1RjeCDQAAcGdZkqDd3Z9I8on13PfwJBtNqJ7NEG7neiL7sj8fAAAAczN1HAAAAEYkaAMAAMCIBG0AAAAYkaANAAAAIxK0AQAAYESCNgAAAIxI0AYAAIARCdoAAAAwIkEbAAAARiRoAwAAwIgEbUiy0/23XeoSAACATYSgDQAAACMStAEAAGBEgjYAAACMSNAGAACAEQnaAAAAMCJBGwAAAEYkaAMAAMCIBG0AAAAYkaANAAAAIxK0AQAAYESCNgAAAIxI0AYAAIARCdoAAAAwIkEbAAAARiRoAwAAwIgEbQAAABiRoA0AAAAjErQBAABgRII2AAAAjEjQBgAAgBEJ2gAAADAiQRsAAABGJGgDAADAiARtAAAAGJGgDQAAACMStAEAAGBEgjYLUlX3qqrXLuL5dq+qi4efS6rquWvVclJVfbuqvlVVjx/W37uqvlRV3x1+/+5i1QsAACBos1D3SjJ10K6JDelnlyVZ1d0rkzw7yUeqasth2/uTnNrdj0iyS5JvDesPTfLl7t4hyZeH1wAAAItC0Gahjkyy/TDC/N6qelNVfb2qLq2qdyRJVa2oqiuq6pOZBOUHVtWNQ/vLq+r/DiPVX62qH1TVPnOdrLtv6u5fDy+3StLDObZN8uQkHxva/aq7fzG02zfJccPycUn+cLZjV9WBVXVBVV1w3XXXbdBFAQAAWEPQZqEOTfL9YYT5S0l2SLJ7kpVJdquqJw/tdkhydHfv2N1XJ7lnkq90945JbkjyriT/Kclzk7xzvhNW1WOr6vIkq5McNATvBye5NsknquqiqvpoVd1z2OW+3f3jYflfktx3tuN29zHdvaq7V2233XbrcSkAAAB+m6DNhnjm8HNRkm8keUQmATtJru7uc2e0/VWSU4fl1Um+1t23Dssr5jtJd583BPTHJHlLVW2VZMskj07yoe7eNckvM8sU8e7uDKPgAAAAi0HQZkNUkiO6e+Xw89Du/tiw7Zdrtb11CL1JcnuSW5Kku2/PJDSvU3d/K8mNSR6V5Jok13T3ecPmkzIJ3kny/6rqfkky/P7Jwt8aAADA+hG0WagbkmwzLJ+W5JVVtXWSVNX9q+r3xjxZVT14zcPPqur3Mxk1v6q7/yXJP1XVw4emT0/yzWH55CQvH5ZfnuRvx6wJAABgPlONJMIa3f3Tqjqrqi5LckqSzyQ5p6qSyWjzS5PcNuIpn5Tk0Kq6NZOR8Nd295onlx2c5PiquluSHyQ5YFh/ZJLPVtWrklyd5AUj1gMAADCv+s1sXth8HX300f3a1y7afw8OAAAsPzVtQ1PHAQAAYESmjrNRqKpnJXn3Wquv7O7nLkU9AAAA60vQZqPQ3adl8nA1AACAZc3UcQAAABiRoA0AAAAjErQBAABgRII2AAAAjEjQBgAAgBEJ2gAAADAiQRsAAABGJGgDAADAiARtAAAAGJGgDQAAACMStAEAAGBEgjYAAACMSNAGAACAEQnaAAAAMCJBGwAAAEYkaAMAAMCIBG0AAAAYkaANAAAAIxK0AQAAYESCNgAAAIxI0AYAAIARCdoAAAAwIkEbAAAARiRoAwAAwIgEbQAAABiRoA0AAAAjErQBAABgRII2AAAAjEjQBgAAgBEJ2gAAADAiQRsAAABGJGgDAADAiARtAAAAGJGgDQAAACMStAEAAGBEgjYAAACMSNAGAACAEQnaAAAAMCJBGwAAAEYkaAMAAMCIqruXugZYcm9+85tvuOtd73rFUtfBpuPGG2/cbuutt75uqetg06FPMTZ9ijHpT4xtI+1T173rXe969jQNBW1IUlUXdPeqpa6DTYc+xdj0KcamTzEm/YmxLfc+Zeo4AAAAjEjQBgAAgBEJ2jBxzFIXwCZHn2Js+hRj06cYk/7E2JZ1n3KPNgAAAIzIiDYAAACMSNAGAACAEQnabFaq6tlVdUVVfa+qDp1l+92r6oRh+3lVtWLxq2Q5maJPvaGqvllVl1bVl6vq95eiTpaPdfWpGe3+qKq6qpbtf33CnW+a/lRVLxj+nLq8qj6z2DWyvEzx996Dqur0qrpo+Ltvr6Wok+Whqj5eVT+pqsvm2F5VddTQ3y6tqkcvdo3rS9Bms1FVWyT5YJLnJHlkkhdV1SPXavaqJD/v7ocmeV+Sdy9ulSwnU/api5Ks6u6dk5yU5D2LWyXLyZR9KlW1TZLXJzlvcStkOZmmP1XVDknekuSJ3b1jkkMWvVCWjSn/jPqzJJ/t7l2T7J/k6MWtkmXm2CTPnmf7c5LsMPwcmORDi1DTKARtNie7J/led/+gu3+V5P8k2XetNvsmOW5YPinJ06uqFrFGlpd19qnuPr27bxpenpvkAYtcI8vLNH9OJcmfZ/JF4L8tZnEsO9P0p1cn+WB3/zxJuvsni1wjy8s0faqT/M6wvG2Sf17E+lhmuvuMJD+bp8m+ST7ZE+cmuVdV3W9xqtswgjabk/sn+acZr68Z1s3aprt/neT6JPdZlOpYjqbpUzO9Kskpd2pFLHfr7FPDtLkHdvcXFrMwlqVp/ox6WJKHVdVZVXVuVc03sgTT9Km3J3lpVV2T5ItJDl6c0thELfTfWhuNLZe6AIDNQVW9NMmqJE9Z6lpYvqrqLkn+MskrlrgUNh1bZjIlc89MZtycUVU7dfcvlrQqlrMXJTm2u/+iqh6f5FNV9ajuvn2pC4PFZESbzcmPkjxwxusHDOtmbVNVW2Yy5emni1Idy9E0fSpV9YwkhyXZp7tvWaTaWJ7W1ae2SfKoJF+tqquSPC7JyR6Ixhym+TPqmiQnd/et3X1lku9kErxhNtP0qVcl+WySdPc5SbZKst2iVMemaKp/a22MBG02J19PskNVPbiq7pbJAzpOXqvNyUlePiw/P8lXursXsUaWl3X2qaraNclHMgnZ7n1kXebtU919fXdv190runtFJvf979PdFyxNuWzkpvl77/OZjGanqrbLZCr5DxazSJaVafrUD5M8PUmq6j9mErSvXdQq2ZScnORlw9PHH5fk+u7+8VIXNQ1Tx9lsdPevq+pPkpyWZIskH+/uy6vqnUku6O6Tk3wskylO38vkwQz7L13FbOym7FPvTbJ1khOH5+r9sLv3WbKi2ahN2adgKlP2p9OSPLOqvpnktiRv6m4zuZjVlH3qvyX531X1p5k8GO0VBi2YS1X9dSZf9m033Nf/tiR3TZLu/nAm9/nvleR7SW5KcsDSVLpwpd8DAADAeEwdBwAAgBEJ2gAAADAiQRsAAABGJGgDAADAiARtAAAAGJGgDQAAACMStAEAAGBE/x9KYAcf5OXT9QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1008x720 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# MGBM var importance\n",
    "mgbm_default.varimp_plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA9oAAAJTCAYAAAD3327jAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xm4JFV9//H3B0ZBFsEgGsToGCDiAo5sKouC4DpuKErUqKBxwX0XJUbwp2EMiUZcQFA2JWpQUQQBRXZQFllmUBANjEZRXEFkU4bv7486F5qm+947Q83cGXi/nqef7q46dc6p6r4wnzqnqlNVSJIkSZKkfqw00x2QJEmSJOnuxKAtSZIkSVKPDNqSJEmSJPXIoC1JkiRJUo8M2pIkSZIk9cigLUmSJElSjwzakiQtgSRnJrmlh3p+keSni1F+wySV5LN3tW1JkrR0GLQlSSuEJEe2gPn6aZT9diu787Lo291NO4lQSbad6b4sbYt7ouOeIMlO7fM/aZIyEyd8fjq0/MFJ3pLkhCQLk9yc5Pftb/J5U7R7ryT/nOQ7SX6T5C/t+dtJXplk1l3Ypx2THJHkJ0mua3X/OslJSd6TZP0R20z8HQw+bklydZJjkzxtxDazBsouSvLQSfp0xkDZf1rSfZO0fFri/2BJkrSMHQy8BPhn4NPjCiWZDewE/Ar45lLsz0uA+yzF+qUV0VuBdwBXACcDVwOzgZ2BpyTZr6rePbxRkocAxwCPAX4NHNue/xZ4JvAU4PVJnlNVV023M0nWBg4HngP8BTi91X0DsC7wOGAesE+Srapq/ohqDgV+3l7fB3gE8AxgbpJXVdUhI7a5he7f2a8C/nVEvzYGth0oJ+luxj9sSdIKoapOTXI58Ngkm1XVBWOKvgoIcGhV3eWp3ZP05+dTl5Lucb4PPLGqzhhcmOTRwNnAu5IcWVUXD6xbAziBLsAeAryxqm4cWL86cCDwT8C3kjxhcP04bQT8a8AOdKH/FVX1ixHlHg18ELjvmKoOqaozh7bZFfgS8L7W52G/BP4IvDLJPlW1aGj9q9vzscCkI/2SVkxOHZckrUgObs+vHrUyycrA7kABnx1Yvn6SDyQ5u00X/UuSX7bp6BuPqOe266CTPDzJUUl+m+TWienUo67RTrJKkjclOT7Jz9q02T+0qbB3mmY6tO3aST6d5KokNyX5YZI3JMl0D06S1ZO8L8nFSa5P8ue2z7tOt44p6v9Fkp8muW+Sj7f3Nya5MMlzWplZSd7fpuje1MrvMaKuienJ/5JkmyTfTfKn9jg+yWZj+rB2ko8M1P+HdNOUnzxFG49P8q1WvpL8U5IC1gc2GJoePPjdeX77nvxk4Jien+SNSe7076gkX2h1/F2S1ye5pPXz10kOTDIyzLXynxjYr98nOTfJXmPKfjrJFbl9avY3kmw+2ee3LFTVV4ZDdlt+CfCV9nb7odXvpAvZpwP/PByiq+p6YDfgHLoR7zdPszsvpwvZlwHPGhWyJ/pWVc+nO0kwXd9uz+tOUuZguu/XMwYXJrl369vpwI8Xo01JKxCDtiRpRXI43fTPFydZbcT6Z9D9w/akqrpyYPkOwLuBPwBfBf4LOBd4EXBuG9Ea5R9auQcDX6D7h/N1k/Rv3Vb3GsB3gI/STYfdHDg+yW5jtluFbsRtJ+C/WzvrAJ9s9U0pyf2As4APA3+lG2U7HHgg8KUke0+nnmlYBTgJeBrwdbrjshHwtSTb0x3f1wCnAJ+jGyX8dJIXjKlv61b2Rrr9PRF4KnBmkq2H9vFvgO9x+2f5X8DRwDbASUn+eUwb29KFmnu3Ph0B/ATYh+7z/GN7PfE4ZmDbfwfm0IWwTwCfb/v0iVbXOP9J91lcCHyKbhr0a+mOzx0keRxwMfBG4BfAx4EvAn9maNpxki2Ai4DX0QXI/ekukdgeODvJU4fKT1wzvNRmdyyGv7bn4b5MnDj7UFXVqA3biPC/tbevmWZ7E/X++3RGwBdzBsxO7fn8ScocSTdFffh7uTNwf24/cSjp7qiqfPjw4cOHjxXmAXyZbsR6txHrvtHW7TK0/IHAGiPKPxa4Hvjm0PINWz0FfHBMP84Ebhlatiqw/oiyawOXAr8FVhla94vWzmnAvQeW3x+4sq3bekTfPjtUzxfa8rcPLb8PXei/Fdhkmsf4zFbXtmP6+vXB/aA7kVF04ff7wFoD6zaiC1jnDdW108Axft3Quhe05ZcBGVj+ubb800PlN6YLzDcBfzemjVeN2ddfAD+d5FhsMGLZSnQhqoDNx3wOVwIPHlh+L7qp0wVsNrB8Fbrrfwt40Yi2huu4gu6kxPBn82C6+xL8Yuh7NKvVfcu4fRzR5sRxuwLYe8xj/1Zm7LEb8TfwW2ARsNHA8oe1ev7C0N/GiDrWaNsX8LdTlL03XaAv4KHT3fcxfweHDOz3R9r3/y/AAmDjoW0mjvfC9v6w9v1fb6DMSXR/K6vSXR9ewD8tSR99+PCx/D4c0ZYkrWgOas93GCVKsh7dTZN+Qxe4b1NVV1fVn4crqqoL6QLujummnQ+7CvjQdDtWVTdV1S9HLL+G7oZK96cb3R5lz6r6y8A2v6MbEYVuOvxYSR4AvBj4flV9dKjtG4E96a5bf/E0d2Uqb6mqmwfaOAX4P+B+wLur6tqBdT+hC9+bjpkGfxnwmaE+f5VudP7hdCPeJFmF7gZ0f6K7Lnaw/GV0o+GrAC8b0cb5VTXZ6PNYVfW/I5bdSjfqDN3I/ij71MBU5ar6K913AGCrgXLPA/4O+FpV/c+ItganOz+HLpj+Vw1dM9zK/QfdjI7tB5bfQjct+1Fj+jmZhwEfGPN403QraZ/7IXTf/0+278SE9drzbwa/U6O0v+E/trcPmqLZ+wMTf9N3+ptM8uQkew89njOmrt25fb/fDTyX7nv433QnIyZzMF343r21+/fAk4EvVNVNU2wraQXmzdAkSSuak4H/BbZJ8oiqurQt353u/2uHtVBzB+0f0a+lC7rrcOf/B/4N3YjboIsGw+90JNkEeBfddOUH0YW/QXf6GSHgZkZfH3pqe37sFM1uRTfKmjFTxCf68Igp6pmO31XVz0Ysv4ouMI66Sd0v6UYYH0B3F+pBZ1TVqOnCp9JNCX8sXeh+JN0I4DntxMWwk+lOKIw6VueOWDYtSe5P93k+ky54rj5UZNTnCaOnFP9fe77fwLLHt+fjp9GdJ7Tnh435nB/enh/B7dcQT5yIWBLfraqdRq1IsiHd9Pvp+DjddOlT6Y7l8uDJwPD175/jjpcNTNhu4sRGu756NvA2uqnsT02yYzv5cidVdVaSS4FXJdmXbjp7cNq4dLdn0JYkrVCqauJmVfvSjWq/o42YvYpuCuad/gGb5B10o31/oJu2+TO66bcFPB/YhDsHYuiuq522JNu0+lcCvks3sn4d3bTtzYBnj2nnt2PC5kT7a03R9Drt+XHtMc4aU9QzHdeOWX4LsGjUzAFuvyb3XiPWDQfvCcP7PvH8qzHlJ5avPUldi6VdE34+8FC6G3EdQfcduoXuxMybGP15Aow6GTBxHAZnT0z0906jriNMfM5T3dyuj8+5F0k+SnecTqG7IdnwiauJz2bdJKtMNqqd7u7jEycppvqJr9/RTTNfme6E1x1+JaCq/gX4l1bv05neiQ5a/y8H9kjyWLrZAy8Ajppks4Pp7tfwNNpN3apqwXTak7TiMmhLklZEh9L9HM/Lk7wX2A74e+DkqvrpYMEk96Kb8nkV3bWxVw+t326SdkbemGkS76cbdd1ueGpvkvfTBe1R1k2SEWH7b9vzuHDL0PqRv1G8nHvgmOXD+37t0PJh6w2VG7S4n+OE19CF7PdX1R0uIWjfm2lPn57ERCAfNzI+aGLf5lbVt3poe6lpJ7/2p7vB20nAc2rEDcmq6ookv6L7/J5Idz+BcZ5MdxLriqqa9ORJVf0lyXl0MwZ25PZp+306h+7E1lZMHrSPoDsxeDDd9/dOd5KXdPfjNdqSpBVOC8vH0F2H+Txuv177oBHFHwisCZw5ImTfl6mnZS+ODemuNT1zxLonTbLdKtw+hXjQ9u35winaPYcuTE520mB5td2Ya7e3b88T+34p3RT7x475iawd2vO431cfZ2LUc5QN2/Od7hTO5J/n4pi4ZOAZk5a6Y9nl+nNun+eBdCH7BODZo0L2gImfU3vfmO8C7afUJq7NH/V3Plm970qy6jS3WRwTo+uT/nu6qn5P93veD6ab4fLlpdAXScsZg7YkaUU1MUX8HXTXf/6O7qeehv2KLqBt2aaeArdda/kJ7ni97F21kG50+g43nkryWrpRtcnMa32a2Ob+3D7yNeloXFX9CvgS8Pgk7x11Y7d0vw3+0Kl3YZnbmO7a+du0nwLbhu43hs+G7kZzdDefWotuNsNg+Y3oQt1f6O76vTh+Dzyg3Wxt2ML2vP1Qe1sA71nMdsb5Ot21289P8sLhlUkePPD26NanN2fM77In2Xo4VCbZOMnDR5XvWwvEn6ObDXAs8Lxp3PRrP7rPenvgMyP6vxrdzdQeT/czaPtPszuH010X/gjgm0nGzRoYdbnBpNpNzZ7b3p46jU3eS/ffqadX97vgku7mnDouSVpRfZsudEzcwfmTo25cVlWLknwCeCewIMkxdCPIT6YLbafR3+jkx+gC9dlJ/ofuzsRb0d3E6qt013KO8gu6UfdLBvq3C9000/2r6uxptL0H3QjsvwG7JTmT7uZu69HdSGwL4IV016cvT44H9k8yl+7nkjaiu27+Rrqf5Bqc9j1xk7m3JNmK7rNbl+730NcA9qiqO1yLOw3fpZvVcEKSM+jC+oVVdRzdTzO9A/hEkp2An9L9tvqz6D7Pqa6VnlJV3dwC9gnA/yQ5he7mbfehC4hPpLscYaLs81vZE5KcRfeb2jcCDwG2pLth27p0P3VGkll0swEWsWz+3bcP3Y0JbwDmA+8dMUh9QVXddtOxqrquXSd9DN3Nwp6V5Hi667f/FphLNzPlAqYeHb9NVd2SZGe63z5/FnBFktOAH7b+rQs8mu7v82a6mSGjvLJ9/tDdZ2A23Uya1YCjq+qb0+jLz1j+/vYkLUUGbUnSCmngpmgT185Odhff99L97Ncr6UZPr6G7FnQvumsn++rTcUme2+r9R7qbX51LN1K3MeOD9s10wX9fup+wWofuzuofBj41zbavbdcNv5buZ7x2oQvsV9PdHfqtdHfmXt6cTbef/4/br3n+DrBXVf1gsGBV/T7J4+imEO8MvJ0uMH2P7vr0k5ag/X2A+9IFse3oppF/Djiuqn7Rjuk8usD7dLrQ+lrgdHoI2gBVdU6SOXTf06fTjeZfRxfs9x4qe2GSTen2/Vl03+lb6WZu/IDuPgF/ZOY8rD2vxtDPsA240929q2phmymwG91xfQ7dSPM1dCcT9gIObz9XNm3tDvXPTvIU4OV0oXobusD8B7rQ/V7g86N+mq8Z/Hm9ortW/gd0114fsjj9kXTPkdE3OZUkSVp62gjhdxhxozFJklZ0XqMtSZIkSVKPDNqSJEmSJPXIoC1JkiRJUo+8RluSJEmSpB5513EJOPzww+sVr3jFTHdDkiRJ0vLrTr9XOI5TxyXg+uuvn+kuSJIkSbqbMGhLkiRJktQjg7YkSZIkST0yaEuSJEmS1CODtiRJkiRJPTJoS5IkSZLUI4O2JEmSJEk9MmhLkiRJktQjg7YkSZIkST0yaEuSJEmS1CODtiRJkiRJPTJoS5IkSZLUI4O2JEmSJEk9MmhLkiRJktQjg7YkSZIkST0yaEuSJEmS1CODtiRJkiRJPTJoS5IkSZLUI4O2JEmSJEk9MmhLkiRJktQjg7YkSZIkST0yaEuSJEmS1CODtiRJkiRJPTJoS5IkSZLUI4O2JEmSJEk9MmhLkiRJktQjg7YkSZIkST0yaEuSJEmS1CODtiRJkiRJPZo10x2QlgcLfnkts/c8bqa7IUmSJAlYOG/uTHfhLnFEW5IkSZKkHhm0JUmSJEnqkUFbkiRJkqQeGbQlSZIkSeqRQVuSJEmSpB4ZtCVJkiRJ6pFBW5IkSZKkHhm0JUmSJEnqkUFbkiRJkqQeGbQlSZIkSeqRQVuSJEmSpB4ZtCVJkiRJ6pFBW5IkSZKkHhm0JUmSJEnqkUFbkiRJkqQeGbQlSZIkSeqRQVuSJEmSpB4ZtCeR5M/LqJ3dkjxoGbV1apItlkVbQ+3OTvKSgfdbJNl/adQ9Yv2M7LMkSZKkeyaD9vJhN2CZBO2lKcmsSVbPBm4Lw1V1flW9uaem71C3JEmSJM0kg/Y0pLNfkkuSLEiya1u+RpLvJrmgLX9uWz47yaVJDk7ywyTfTnKfMXXvAmwBHJnkoiT3SbJjkgtbnYckWWXMtk9PctTA++2THNteH5Dk/Nb+PmO2//PA612SHNZer5vkq0nOa49tJjk2eyf5fJKzgM+3fT+jHZMLkmzdis4Dtmv7+Lahvv5Nkq8nmZ/k+0k2naS9J7U6LmrHaM0Rdd8nyZfaZ3A0MO7Yv6Ydo/MX3XDtuCYlSZIkabEYtKfn+cAc4DHATsB+SdYDbgJ2rqrNgB2A/0ySts1GwKeq6lHANcALRlVcVV8BzgdeWlVzgAIOA3atqk2AWcAeY/p1EvC4JKu397sCX2qv96qqLYBNgSdNFl5H+DjwsarasvX7s1OUfySwU1W9GPgN8JR2THYFJqaH7wmcUVVzqupjQ9vvA1xYVZsC7wOOmKStdwJvaMdqO+DGEXXvAdxQVY8APgBsPqqiqjqoqraoqi1WXm2tKXZRkiRJkqbHoD092wJfrKpFVXU1cBqwJRDg35LMpwu96wMPbNtcWVUXtdc/oJvePB0Pb9te3t4fDjxxVMGqugU4AXh2m7Y9F/hGW/2iJBcAFwKPogvD07UT8MkkFwHHAPdNssYk5Y+pqhvb63sBBydZABw1zXa3BT7f9ulkYJ0k9x1T9izgo0neDKzdjsGwJwJfaPXNB+ZPow+SJEmS1IvJrqnV1F4KrAtsXlV/TbIQWLWtu3mg3CLGTF/uwZeANwJ/AM6vquuSPIxu5HfLqvpjmxK+6ohta+D14PqVgMdX1U3T7MP1A6/fBlxNN/q/Et2of2+qal6S44BnAmcleVqf9UuSJEnSXeWI9vScAeyaZOUk69KNmJ4LrAX8poXsHYCHLmH91wFrttc/BmYn2bC9fxndCPo4pwGbAa/m9mnj96ULv9cmeSDwjDHbXp3kEUlWAnYeWP5t4E0Tb5LMWYx9WQv4VVXd2vq+cls+uI/DzqA7aUGS7YHfVdWfRhVMskFVLaiqjwDnARuPqPt02s3Rkjyabvq8JEmSJC0TBu3pOZpu+vHFwMnAu6vq18CRwBZtmvTLgcuWsP7DgAPbVO0AuwNHtXpvBQ4ct2FVLQKOpQvTx7ZlF9NNGb8M+G+66daj7Nm2ORv41cDyN7f9mp/kR8DrFmNfPg28IsnFdCF4YrR7PrAoycVJ3ja0zd7A5m0K/jzgFZPU/9Z2U7r5wF+B40fUfQCwRpJLgQ/STd2XJEmSpGUiVTV1Kelubo+99q3jFznwLUmSJC0PFs6bO9NdGCVTF+k4oi1JkiRJUo+8GdoylORTwPBvUn+8qg6dxrZHAw8bWvyeqjqxr/5N0f7uwFuGFp9VVW+4O7QnSZIkSX0xaC9DdyUkVtXOU5daetrJgClPCKyo7UmSJElSX5w6LkmSJElSjwzakiRJkiT1yKAtSZIkSVKPDNqSJEmSJPXIoC1JkiRJUo8M2pIkSZIk9cigLUmSJElSjwzakiRJkiT1yKAtSZIkSVKPZs10B6TlwSbrr8UBr587092QJEmSdDfgiLYkSZIkST0yaEuSJEmS1CODtiRJkiRJPTJoS5IkSZLUI4O2JEmSJEk9MmhLkiRJktQjg7YkSZIkST0yaEuSJEmS1CODtiRJkiRJPZo10x2QlgcLfnkts/c8bqa7IWk5tXDe3JnugiRJWoE4oi1JkiRJUo8M2pIkSZIk9cigLUmSJElSjwzakiRJkiT1yKAtSZIkSVKPDNqSJEmSJPXIoC1JkiRJUo8M2pIkSZIk9cigLUmSJElSjwzakiRJkiT1yKAtSZIkSVKPDNqSJEmSJPXIoC1JkiRJUo8M2pIkSZIk9cigLUmSJElSjwzakiRJkiT1yKAtSZIkSVKPDNr3QElmJ3nJMmprtySfXAbtvDXJajPZB0mSJEkCg/ZyK8nKS6neWcBsYJkE7WXorcDIoC1JkiRJy5JBewa0EeXLkhyZ5NIkX0myWpKFST6S5ALghUnmJPl+kvlJjk5yv7b9qUk+nuSiJJck2aotXz3JIUnOTXJhkue25bslOSbJycB3gXnAdm37tyU5Pcmcgf6dmeQxI/q9Uuvj2gPLfpLkgUmeneSc1u5JSR44YvvDkuwy8P7PA6/fleS8tq/7THLsVk9yXJKL277vmuTNwIOAU5Kc0srtnuTyJOcC24yp6zVJzk9y/qIbrh3XpCRJkiQtFoP2zHk48OmqegTwJ+D1bfnvq2qzqvoScATwnqraFFgAfGBg+9Wqak7b7pC2bC/g5KraCtgB2C/J6m3dZsAuVfUkYE/gjKqaU1UfAz4H7AaQ5B+AVavq4uEOV9WtwDeAnVvZxwE/q6qrgTOBx1fVY4EvAe+e7oFI8lRgI2ArYA6weZInjin+dOCqqnpMVT0aOKGq9geuAnaoqh2SrAfsQxewtwUeOaqiqjqoqraoqi1WXm2t6XZXkiRJkiZl0J45/1dVZ7XXX6ALhABfBkiyFrB2VZ3Wlh8ODIbPLwJU1enAfdso81OBPZNcBJwKrAo8pJX/TlX9YUxfjgKeleRewCuBwybp95eBXdvrf5zoL/Bg4MQkC4B3AY+apI5hT22PC4ELgI3pgvcoC4CntJH/7apq1FD044BTq+q3VfWXgT5KkiRJ0lJn0J45Neb99Xdh+wAvaCPVc6rqIVV16VT1VtUNwHeA5wIvAo6cpN3vARsmWRd4HvC1tvwTwCerahPgtXQhf9gttO9ckpWAe7flAfYd6PeGVfW5MX29nG50fgHwoST/OklfJUmSJGmZM2jPnIckeUJ7/RK6qde3aSO1f0yyXVv0MuC0gSK7AiTZFri2lT8ReFOStHWPHdP2dcCaQ8s+C+wPnFdVfxzX6aoq4Gjgo8ClVfX7tmot4Jft9SvGbL4Q2Ly9fg5wr/b6ROCVSdZo/V4/yQNGVZDkQcANVfUFYD+60D28T+cAT0qyThulf+G4/ZEkSZKkvs2a6Q7cg/0YeEOSQ4AfAQcAbxoq8wrgwPazVVcAuw+suynJhXRh9ZVt2f8D/guY30aMrwSeNaLt+cCiJBcDh1XVx6rqB0n+BBw6jb5/GTiPdl13szdwVJI/AicDDxux3cHAN1q7J9BG2avq20keAXyvnSP4M/BPwG9G1LEJ3bXntwJ/BfZoyw8CTkhyVbtOe2+60fdrgIumsU+SJEmS1It0A5RalpLMBo5tN/Naku1PBd5ZVef32KcH0V3XvXG76dk9yh577VvHL9p0prshaTm1cN7cme6CJEmaeZluQaeOiyQvp5tuvdc9MWRLkiRJUp+cOj4DqmohsESj2W377XvrTFffEXQ/JXabJLsDbxkqelZVvaHPtsdJsg7db34P23HgunBJkiRJWu4YtDVSVR3K9K7XXlrt/57uN7UlSZIkaYXi1HFJkiRJknpk0JYkSZIkqUcGbUmSJEmSemTQliRJkiSpRwZtSZIkSZJ6ZNCWJEmSJKlHBm1JkiRJknpk0JYkSZIkqUcGbUmSJEmSemTQliRJkiSpR7NmugPS8mCT9dfigNfPneluSJIkSbobcERbkiRJkqQeGbQlSZIkSeqRQVuSJEmSpB4ZtCVJkiRJ6pFBW5IkSZKkHhm0JUmSJEnqkUFbkiRJkqQeGbQlSZIkSeqRQVuSJEmSpB7NmukOSMuDBb+8ltl7HjfT3ZCWmoXz5s50FyRJku4xHNGWJEmSJKlHBm1JkiRJknpk0JYkSZIkqUcGbUmSJEmSemTQliRJkiSpRwZtSZIkSZJ6ZNCWJEmSJKlHBm1JkiRJknpk0JYkSZIkqUcGbUmSJEmSemTQliRJkiSpRwZtSZIkSZJ6ZNCWJEmSJKlHBm1JkiRJknpk0JYkSZIkqUcGbUmSJEmSenSPD9pJ9k7yzknWH5ZklxHL5yR55hR1b59k6yXs1+uSvHxJtl0eJXlrktUG3n8rydpLo+6hdbsl+WQf7UiSJEnSdNzjg/ZdMAeYNGgD2wNLFLSr6sCqOmJJtp0J6Uz2fXorcFsYrqpnVtU1PTV/h7olSZIkaSbdI4N2kr2SXJ7kTODhbdkGSU5I8oMkZyTZeGCTnZKc37Z5VpJ7Ax8Edk1yUZJdR7QxG3gd8LZWZrsks5OcnGR+ku8mecgkfbxtpD3JqUk+kuTc1oft2vKVk/xHkktanW9qy3dMcmGSBUkOSbJKW74wyb6tP+cn2SzJiUn+N8nrBtp+V5LzWp37TNLH2Ul+nOQI4BLg75Ic0Or+4cS2Sd4MPAg4JckpA325f3v99rYPlyR56yTtrZ7kuCQXt7K7jql793aczgW2maS+17S+nr/ohmvHFZMkSZKkxXKPC9pJNgf+kdtHpLdsqw4C3lRVmwPvBD49sNlsYCtgLnAg3XH7V+DLVTWnqr483E5VLWxlP9bKnAF8Aji8qjYFjgT2X4yuz6qqrehGbz/Qlr2m9W3ORJ1JVgUOA3atqk2AWcAeA/X8vKrmAGe0crsAjwcmQvFTgY3a/s4BNk/yxEn6tRHw6ap6VFX9DNirqrYANgWelGTTqtofuArYoap2GNy4fR67A49r/Xh1kseOaevpwFVV9ZiqejRwwnDdSdZr+7INsC3wyHEdr6qDqmqLqtpi5dXWmmQXJUmSJGn67nFBG9gOOLqqbqiqPwHHAKvSTfE+KslFwGeA9Qa2+Z+qurWqfgJcAWw8XOk0PQH47/b683RBcLq+1p5/QBeuAXYCPlNVtwBU1R/oRuivrKrLW5nDgcGgfEx7XgCcU1XXVdVvgZvbNdNPbY8LgQvo9nWjSfr1s6r6/sD7FyW5oG3/KCYJus22dJ/H9VX157af240puwB4Shvd366qRg1DPw44tap+W1X2lqK1AAAgAElEQVR/Ae50EkSSJEmSlqZZM92B5cRKwDVtpHeUmuL9snBze17EXfvcJuq5deD1xPtZQIB9q+oz06zv+okXSR5GNxtgy6r6Y5LD6E5i9KKqLk+yGd1MhA8l+W5VfbCv+iVJkiSpD/fEEe3TgecluU+SNYFnAzcAVyZ5Idx2Y6/HDGzzwiQrJdkA+Hvgx8B1wJpTtDVc5my6aesAL6Wbvn1XfAd4bZJZrd9/0/o2O8mGrczLgNMWo84TgVcmWaPVuX6SB0xz2/vSBe9rkzwQeMbAunHH6wy6z2O1JKsDOzPmuCR5EHBDVX0B2A/YbETd59BNWV8nyb2AF06z75IkSZLUi3tc0K6qC+imE18MHA+c11a9FHhVkouBHwLPHdjs58C5rfzrquom4BTgkeNuhtZ8E9h54mZowJuA3ZPMpwvAb7mLu/PZ1rf5rd8vaX3bnW4a/AK6keoDp1thVX2bbnr799r2X2HqEwoT215MN2X8slbHWQOrDwJOmLhh2cA2F9BdK34uXUj+bFVdOKaJTYBz2/T+DwAfGq67qn4F7A18r7V/6XT6LkmSJEl9SdVMzIKWli977LVvHb9o05nuhrTULJw3d6a7IEmStKLLdAve40a0JUmSJElamrwZWg+S7M6dp4GfVVVvmMa2e3Hn64iPqqoP99W/uyrJOsB3R6zasap+v6K3J0mSJEl9Mmj3oKoOBQ5dwm0/DCw3oXqUFm7H3ZF9hW9PkiRJkvrk1HFJkiRJknpk0JYkSZIkqUcGbUmSJEmSemTQliRJkiSpRwZtSZIkSZJ6ZNCWJEmSJKlHBm1JkiRJknpk0JYkSZIkqUcGbUmSJEmSemTQliRJkiSpR7NmugPS8mCT9dfigNfPneluSJIkSbobcERbkiRJkqQeGbQlSZIkSeqRQVuSJEmSpB4ZtCVJkiRJ6pFBW5IkSZKkHhm0JUmSJEnqkUFbkiRJkqQeGbQlSZIkSeqRQVuSJEmSpB7NmukOSMuDBb+8ltl7HjfT3bjHWjhv7kx3QZIkSeqNI9qSJEmSJPXIoC1JkiRJUo8M2pIkSZIk9cigLUmSJElSjwzakiRJkiT1yKAtSZIkSVKPDNqSJEmSJPXIoC1JkiRJUo8M2pIkSZIk9cigLUmSJElSjwzakiRJkiT1yKAtSZIkSVKPDNqSJEmSJPXIoC1JkiRJUo8M2pIkSZIk9cigLUmSJElSjwzakiRJkiT1yKDdkyRnT6PMW5Ostiz6M9Dm85I8clm2OdT+lMdlqPxhSXZZzG3G7mOS2UkuWZz6JEmSJOmuMGj3pKq2nkaxtwIjg3aSlZe07Sm2fR6wzIN2klkw7eNyV83IPkqSJEnSKAbtniT5c3vePsmpSb6S5LIkR6bzZuBBwClJTpnYJsl/JrkYeEKSHZNcmGRBkkOSrDJJewuTfCTJBcALk7w6yXlJLk7y1SSrJdkaeA6wX5KLkmzQHick+UGSM5JsPEkbhyU5MMn5SS5P8qy2fOUk+7X25id57cC+n5HkGOBHQ8clbZtL2v7tOrD8k0l+nOQk4AFTHOd5SX7U2v2PMfu4eTsOFwNvmKSu17R9O3/RDddO1qwkSZIkTdusme7A3dRjgUcBVwFnAdtU1f5J3g7sUFW/a+VWB86pqnckWRX4CbBjVV2e5AhgD+C/Jmnn91W1GUCSdarq4Pb6Q8CrquoTLfQeW1Vfaeu+C7yuqn6S5HHAp4EnT9LGbGArYAO6kwQbAi8Hrq2qLdvJgLOSfLuV3wx4dFVdOVTP84E5wGOA+wPnJTkdeALwcLoR6QfSBfRDRnUkyTrAzsDGVVVJ1q6qa0bs43zgjVV1epL9xu1YVR0EHASwx177FosmOQqSJEmSNE2OaC8d51bVL6rqVuAiurA6yiLgq+31w4Erq+ry9v5w4IlTtPPlgdePbqPJC4CX0gX9O0iyBrA1cFSSi4DPAOtN0cb/VNWtVfUT4ApgY+CpwMtbHecA6wAbtfLnjgjZANsCX6yqRVV1NXAasGXbx4nlVwEnT9KXa4GbgM8leT5ww4h9XBtYu6pOb4s+P8X+SZIkSVKvHNFeOm4eeL2I8cf5pqq6K+Oo1w+8Pgx4XlVdnGQ3YPsR5VcCrqmqOYvRRo14H+BNVXXi4Iok2w/1qVdVdUuSrYAdgV2ANzL5aLwkSZIkLXOOaC9b1wFrjln3Y2B2m5oN8DK6Ud/pWhP4VZJ70Y1o36nNqvoTcGWSF8Jt10c/Zop6X5hkpSQbAH/f+nkisEdriyT/kGT1Keo5A9i1Xd+9Lt1I9rnA6QPL1wN2GFdBG5Ffq6q+BbyNbhr68D5eA1yTZNu27qV3qkiSJEmSliKD9rJ1EHDCxM3QBlXVTcDudNO6FwC3AgcuRt3vp5vGfRZw2cDyLwHvajdZ24AueL6q3Sjsh8Bzp6j353SB+Hi6a7tvAj5Ldy31Bel+OuszTD074mhgPnAx3fTwd1fVr9vyn7T6jgC+N0kdawLHtmuwzwTePmYfdwc+1aa2Z4p+SZIkSVKvUjU8M1jqJDmMgZuM3Z3tsde+dfyiTWe6G/dYC+fNnekuSJIkSVOZ9iCeI9qSJEmSJPXIm6Et55IcDTxsaPF7hm9Edhfb2At44dDio6pqt77aWFzLYr8lSZIkaWkwaC/nqmrnZdDGh4EPL+12Fsey2G9JkiRJWhqcOi5JkiRJUo8M2pIkSZIk9cigLUmSJElSjwzakiRJkiT1yKAtSZIkSVKPDNqSJEmSJPXIoC1JkiRJUo8M2pIkSZIk9cigLUmSJElSjwzakiRJkiT1aNZMd0BaHmyy/loc8Pq5M90NSZIkSXcDjmhLkiRJktQjg7YkSZIkST0yaEuSJEmS1CODtiRJkiRJPTJoS5IkSZLUI4O2JEmSJEk9MmhLkiRJktQjg7YkSZIkST0yaEuSJEmS1KNZM90BaXmw4JfXMnvP42a6GyukhfPmznQXJEmSpOWKI9qSJEmSJPXIoC1JkiRJUo8M2pIkSZIk9cigLUmSJElSjwzakiRJkiT1yKAtSZIkSVKPDNqSJEmSJPXIoC1JkiRJUo8M2pIkSZIk9cigLUmSJElSjwzakiRJkiT1yKAtSZIkSVKPDNqSJEmSJPXIoC1JkiRJUo8M2pIkSZIk9cigLUmSJElSjwzay7Ekf57pPixLSWYneckSbLd2ktdPsv6wJLvctd5JkiRJ0vQYtLU8mQ0sdtAG1gbGBm1JkiRJWpYM2iuAdPZLckmSBUl2bcvXSPLdJBe05c9ty2cnuTTJwUl+mOTbSe4zSf2vTnJekouTfDXJam35YUkOSPL9JFck2T7JIa3uwwa2f3Fr/5IkHxlY/ueB17tMbNPq3T/J2a3eidHmecB2SS5K8rYxfX1UknNbmflJNmrbbdCW7deO1yeT/DjJScADxtT1miTnJzl/0Q3XTuOTkCRJkqSpGbRXDM8H5gCPAXYC9kuyHnATsHNVbQbsAPxnkrRtNgI+VVWPAq4BXjBJ/V+rqi2r6jHApcCrBtbdD3gC8DbgGOBjwKOATZLMSfIg4CPAk1sft0zyvGns03rAtsCz6IIywJ7AGVU1p6o+Nma71wEfr6o5wBbAL9p2/9u2exewM/Bw4JHAy4GtR1VUVQdV1RZVtcXKq601jS5LkiRJ0tQM2iuGbYEvVtWiqroaOA3YEgjwb0nmAycB6wMPbNtcWVUXtdc/oJuWPc6jk5yRZAHwUrogPeGbVVXAAuDqqlpQVbcCP2x1bgmcWlW/rapbgCOBJ05jn75eVbdW1Y8G+jwd3wPel+Q9wEOr6sYRZZ7I7cfrKuDkxahfkiRJku4Sg/aK7aXAusDmbYT3amDVtu7mgXKLgFmT1HMY8Maq2gTYZ6COwXpuHarz1inqBKiB16sOrRusK0xTVf038BzgRuBbSZ483W0lSZIkaVkwaK8YzgB2TbJyknXpRmzPBdYCflNVf02yA/DQJax/TeBXSe5FF94Xx7nAk5LcP8nKwIvpRtwBrk7yiCQr0U3nnsp1rS9jJfl74Iqq2h/4BrDpiO1O5/bjtR7dtHpJkiRJWiamGpHU8uFouuukL6YbJX53Vf06yZHAN9uU7/OBy5aw/vcD5wC/bc+Tht1BVfWrJHsCp9CNTB9XVd9oq/cEjm31ng+sMUV184FFSS4GDhtznfaLgJcl+Svwa+DfquoPSc5KcglwPPBuumvGfwT8nG66uSRJkiQtE+kuv5Xu2fbYa986ftGmM92NFdLCeXNnuguSJEnSsjDtS16dOi5JkiRJUo+cOn4PkuRTwDZDiz9eVYfORH8mk+RpdD8bNujKqprOtd6SJEmSNGMM2vcgVfWGme7DdFXVicCJM90PSZIkSVpcTh2XJEmSJKlHBm1JkiRJknpk0JYkSZIkqUcGbUmSJEmSemTQliRJkiSpRwZtSZIkSZJ6ZNCWJEmSJKlHBm1JkiRJknpk0JYkSZIkqUcGbUmSJEmSejRrpjsgLQ82WX8tDnj93JnuhiRJkqS7AUe0JUmSJEnqkUFbkiRJkqQeGbQlSZIkSeqRQVuSJEmSpB4ZtCVJkiRJ6pFBW5IkSZKkHhm0JUmSJEnqkUFbkiRJkqQeGbQlSZIkSerRrJnugLQ8WPDLa5m953FLrf6F8+YutbolSZIkLV8c0ZYkSZIkqUcGbUmSJEmSemTQliRJkiSpRwZtSZIkSZJ6ZNCWJEmSJKlHBm1JkiRJknpk0JYkSZIkqUcGbUmSJEmSemTQliRJkiSpRwZtSZIkSZJ6ZNCWJEmSJKlHBm1JkiRJknpk0JYkSZIkqUcGbUmSJEmSemTQliRJkiSpRwZtSZIkSZJ6ZNCWJEmSJKlHBu2eJPlgkp2mKLN9kq2XVZ9am7OTvGRZtjnU/pTHZaj89kmOXcw2Jt3HJKcm2WJx6pQkSZKkJWXQ7klV/WtVnTRFse2BkUE7yawlbXuKbWcDMxK0k6w8zeNyV81mhvZRkiRJkoYZtLltRPTSJAcn+WGSbye5T5I5Sb6fZH6So5Pcb5I6DkuyS3u9MMk+SS5IsiDJxklmA68D3pbkoiTbtW0OTHIO8O9J/ibJ11t730+y6STt7Z3k80nOAj7f9uGM1uYFAyPn84DtWptvS7Jykv2SnNfaee0kbWyf5PQkxyX5cevrSm3dU5N8r7V1VJI1Bvb9I0kuAF44dFx2THJhOyaHJFmlLX96ksvaNs+f4rN6UtuXi1pda47Yx/sk+VL7TI8G7jOmrtckOT/J+YtuuHayZiVJkiRp2gzat9sI+FRVPQq4BngBcATwnqraFFgAfGAx6vtdVW0GHAC8s6oWAgcCH6uqOVV1Riv3YGDrqno7sA9wYWvvfa39yTwS2KmqXgz8BnhKa3NXYP9WZk/gjNbmx4BXAddW1ZbAlsCrkzxskja2At7U2toAeH6S+wP/0treDDgfePvANr+vqs2q6ksTC5KsChwG7FpVmwCzgD3a8oOBZwObA387xT6/E3hDVc0BtgNuHLGPewA3VNUj6D6zzUdVVFUHVdUWVbXFyqutNUWzkiRJkjQ9Bu3bXVlVF7XXP6ALlWtX1Wlt2eHAExejvq8N1DV7knJHVdWi9npb4PMAVXUysE6S+06y7TFVdWN7fS/g4CQLgKPogvEoTwVenuQi4BxgHbqTDOOcW1VXtD5+sfXx8a3+s1o9rwAeOrDNl0fU83C6Y3x5ez9xPDduy39SVQV8YZK+AJwFfDTJm+k+n1tGlHniRD1VNR+YP0WdkiRJktSbJb4u+G7o5oHXi4C1e6pvEZMf5+vvQhuD274NuBp4DN0JlJvGbBPgTVV14jTbqBHvA3ynjaRP1a9eVdW8JMcBz6QL+k9bWm1JkiRJ0pJwRHu8a4E/JtmuvX8ZcNok5afjOmDNSdafAbwUuuuj6aaf/2mada8F/KqqbqXr68pj2jyRbsr2vVo7/5Bk9Unq3SrJw9q12bsCZwLfB7ZJsmGrY/Uk/zBF/34MzJ7YhtuP52Vt+QZt+bjwTmtrg6paUFUfAc6jGxEf3sfTaTdHS/JoYOy17pIkSZLUN0e0J/cK4MAkqwFXALvfxfq+CXwlyXPprnsetjdwSJL5wA2t/en6NPDVJC8HTuD2UeX5wKIkF9NdI/1xuqnsFyQJ8FvgeZPUex7wSWBD4BTg6Kq6NcluwBcnbmhGd8325aOrgKq6KcnuwFHtLunnAQdW1c1JXgMcl+QGupMNk52MeGuSHYBbgR8Cx7fXg/t4AHBokkuBS+mm70uSJEnSMpHusljpztqo+jur6lkz3ZelbY+99q3jFy29ge+F8+YutbolSZIkLROZbkGnjkuSJEmS1COnji+mJJ8Cthla/PGqOnQptbc78JahxWdV1Rt6bGMT2t3OB9xcVY8DTu2rncWxLPZbkiRJkpYGg/ZiWtZBrwX4pRLiB9pYAMxZmm0srmWx35IkSZK0NDh1XJIkSZKkHhm0JUmSJEnqkUFbkiRJkqQeGbQlSZIkSeqRQVuSJEmSpB4ZtCVJkiRJ6pFBW5IkSZKkHhm0JUmSJEnqkUFbkiRJkqQezZrpDkjLg03WX4sDXj93prshSZIk6W7AEW1JkiRJknpk0JYkSZIkqUcGbUmSJEmSemTQliRJkiSpRwZtSZIkSZJ6ZNCWJEmSJKlHBm1JkiRJknpk0JYkSZIkqUcGbUmSJEmSejRrpjsgLQ8W/PJaZu953F2uZ+G8uT30RpIkSdKKzBFtSZIkSZJ6ZNCWJEmSJKlHBm1JkiRJknpk0JYkSZIkqUcGbUmSJEmSemTQliRJkiSpRwZtSZIkSZJ6ZNCWJEmSJKlHBm1JkiRJknpk0JYkSZIkqUcGbUmSJEmSemTQliRJkiSpRwZtSZIkSZJ6ZNCWJEmSJKlHBm1JkiRJknpk0JYkSZIkqUcGbUmSJEmSemTQ1m2SnD3TfVgSSeYkeeYk6xcmuf+y7JMkSZKkey6Dtm5TVVvPdB+W0BxgbNCWJEmSpGXJoK3bJPlze94+yalJvpLksiRHJklbt2WSs5NcnOTcJGsmWTXJoUkWJLkwyQ6t7G7J/2/v3qMtKcs7Af9eaZVBCEaduBzUtCLqgNykxfsl6ngjA9GIeBsVHQlqGImjSxyy1BgZUCZxxCjKeAFvGcRlDIkCcQCFgKAgQtMqXrgYjBnvPSAGsXnnj12tx/ac0/t0F+f06X6etc7q2lVfVb279rdO929/X1XXJ6vqM8Oo8h9X1auGNhdV1V2GdrtW1ZlVdWlVnV9VD5ynxoOr6srh/OdV1R2SvCnJIVX15ao6pKruWlX/UFVrquq9SWqOYx1WVZdU1SXrblo78tUEAAC2VYI2c9k3yZFJdk9y3ySPHELtqUle2d17J3likp8leUWS7u49kzwnySlVtf1wnAcleUaShyQ5JslN3b1vks8necHQ5qQkR3T3fkleneRd89T1+iRPHs5/YHf/fFh3anfv092nJnlDkn/s7j2S/E2Se892oO4+qbtXdfeq7XbYeaHXBwAAYFYrlroAtlhf6O7rk6SqvpxkZZK1Sb7b3V9Mku7+f8P2RyV5x7Dua1V1XZL7D8c5t7tvSHJDVa1N8nfD+tVJ9qqqHZM8Islpw6B5ktxxnrouSHJyVX0sySfmaPOYTMJ9uvtTVfXjhbxxAACAzSFoM5ebZyyvy6b3lZnHuXXG61uHY94uyU+6e59pDtbdh1fVQ5MckOTSqtpvE+sCAAC4TZg6zkJcleQeVfWQJBnuz16R5PwkzxvW3T+TqdpXTXPAYVT8mqo6eNi/qmrvudpX1a7dfXF3vz7J95PcK8kNSXaa0ey8JM8d2j81yW8v6F0CAABsBkGbqQ33Qx+S5B1VdXmSzyTZPpN7qm9XVaszuYf7Rd1989xH+g3PS/KS4Zhrkhw0T9vjh4euXZnkwiSXJzk3ye7rH4aW5M+SPKaq1mQyhfzbC3qjAAAAm6G6e6lrgCX3sqOP7TPW7bXZx7n2uANGqAYAANgCzfq/Gc3GiDYAAACMyMPQ2CJV1dFJDt5g9WndfcxS1AMAADAtQZst0hCohWoAAGDZMXUcAAAARiRoAwAAwIgEbQAAABiRoA0AAAAjErQBAABgRII2AAAAjEjQBgAAgBEJ2gAAADAiQRsAAABGJGgDAADAiFYsdQGwJdhzl51z4ssPWOoyAACArYARbQAAABiRoA0AAAAjErQBAABgRII2AAAAjEjQBgAAgBEJ2gAAADAiQRsAAABGJGgDAADAiARtAAAAGNGKpS4AtgSrv7M2K4/61Kzbrj3ugEWuBgAAWM6MaAMAAMCIBG0AAAAYkaANAAAAIxK0AQAAYESCNgAAAIxI0AYAAIARCdoAAAAwIkEbAAAARiRoAwAAwIgEbQAAABiRoA0AAAAjErQBAABgRII2AAAAjEjQBgAAgBEJ2gAAADAiQRsAAABGJGizIFV156p6+SKeb/+q+vLwc3lVPX2DWj5eVV+rqq9W1cOH9Xepqs9U1TeGP397seoFAAAQtFmoOyeZOmjXxOb0syuTrOrufZI8Jcl7qmrFsO3tSc7s7gcm2TvJV4f1RyU5u7t3S3L28BoAAGBRCNos1HFJdh1GmI+vqtdU1Rer6oqq+rMkqaqVVXVVVX0wk6B8r6q6cWi/pqr+zzBS/dmqurqqDpzrZN19U3f/Yni5fZIezrFzkscked/Q7ufd/ZOh3UFJThmWT0nyByNfAwAAgDkJ2izUUUm+NYwwfybJbkn2T7JPkv2q6jFDu92SvKu79+ju65LcKck53b1HkhuSvDnJf0jy9CRvmu+EVfXQqlqTZHWSw4fgfZ8k30/ygaq6rKreW1V3Gna5e3d/d1j+lyR3n+O4h1XVJVV1ybqb1m7CpQAAAPhNgjab40nDz2VJvpTkgZkE7CS5rrsvmtH250nOHJZXJ/lcd98yLK+c7yTdffEQ0B+S5HVVtX2SFUkenOTE7t43yU8zyxTx7u4Mo+CzbDupu1d196rtdth5ircLAACwcYI2m6OSHNvd+ww/9+vu9w3bfrpB21uG0Jsktya5OUm6+9ZMQvNGdfdXk9yY5EFJrk9yfXdfPGz+eCbBO0n+b1XdI0mGP7+38LcGAACwaQRtFuqGJDsNy2cleXFV7ZgkVbVLVf3OmCerqvusf/hZVf1uJqPm13b3vyT5p6p6wND0CUm+MiyfnuSFw/ILk/ztmDUBAADMZ6qRRFivu39YVRdU1ZVJzkjy0SSfr6pkMtr8/CTrRjzlo5IcVVW3ZDIS/vLu/sGw7YgkH6mqOyS5Osmhw/rjknysql6S5LokzxqxHgAAgHnVr2bzwrbrZUcf22es22vWbdced8AiVwMAAGyBatqGpo4DAADAiEwdZ4tQVU9O8pYNVl/T3U9finoAAAA2laDNFqG7z8rk4WoAAADLmqnjAAAAMCJBGwAAAEYkaAMAAMCIBG0AAAAYkaANAAAAIxK0AQAAYESCNgAAAIxI0AYAAIARCdoAAAAwIkEbAAAARrRiqQuALcGeu+ycE19+wFKXAQAAbAWMaAMAAMCIBG0AAAAYkaANAAAAIxK0AQAAYESCNgAAAIxI0AYAAIARCdoAAAAwIkEbAAAARiRoAwAAwIhWLHUBsCVY/Z21WXnUp35t3bXHHbBE1QAAAMuZEW0AAAAYkaANAAAAIxK0AQAAYESCNgAAAIxI0AYAAIARCdoAAAAwIkEbAAAARiRoAwAAwIgEbQAAABiRoA0AAAAjErQBAABgRII2AAAAjEjQBgAAgBEJ2gAAADAiQRsAAABGJGgDAADAiARtRldVx1fVmqo6/jY49nuravexjwsAADCWFUtdAFulw5LcpbvXTdO4qlZ09y+madvd/3mzKgMAALiNGdHm11TVC6rqiqq6vKo+VFUrq+qcYd3ZVXXvod3JVXVCVV1YVVdX1TOH9acn2THJpVV1yEb2f3dVXZzkrVX1xqo6parOr6rrquoZVfXWqlpdVWdW1e2H/T5bVauG5Rur6pih1ouq6u7D+l2H16ur6s1VdeMSXEoAAGAbJWjzS1W1R5I/TfL47t47ySuTvCPJKd29V5KPJDlhxi73SPKoJL+f5Lgk6e4Dk/ysu/fp7lM3sv89kzyiu181vN41yeOTHJjkw0nO7e49k/wsyQGzlHynJBcNtZ6X5KXD+rcnefuw7/XzvN/DquqSqrpk3U1rN36BAAAApiBoM9Pjk5zW3T9Iku7+UZKHJ/nosP1DmQTr9T7Z3bd291eS3H2OY863/2kbTC8/o7tvSbI6yXZJzhzWr06ycpZj/zzJ3w/Ll85o8/Akpw3LH80cuvuk7l7V3au222HnuZoBAAAsiKDN5rh5xnJtwv4/ne143X1rklu6u4f1t2b25wnMbLNujjYAAACLStBmpnOSHFxVd02SqrpLkguTPHvY/rwk5y/wmJu7/6a4KMkfDsvPnq8hAADA2IwA8kvdvaaqjknyuapal+SyJEck+UBVvSbJ95McusDDbu7+m+LIJB+uqqMzmX7uBmwAAGDR1K9m3sLWoap2yOSBbF1Vz07ynO4+aL59Xnb0sX3Gur1+bd21x832/DUAAGAbNfXtska02Rrtl+SvqqqS/CTJi5e4HgAAYBsiaLPV6e7zk+y91HUAAADbJg9DAwAAgBEJ2gAAADAiQRsAAABGJGgDAADAiARtAAAAGJGgDQAAACMStAEAAGBEgjYAAACMSNAGAACAEQnaAAAAMCJBGwAAAEYkaAMAAMCIVix1AbAl2HOXnXPiyw9Y6jIAAICtgBFtAAAAGJGgDQAAACMStAEAAGBEgjYAAACMSNAGAACAEQnaAAAAMCJBGwAAAEYkaAMAAMCIBG0AAAAYkaANSVZ/Z+1SlwAAAGwlBG0AAAAYkaANAAAAIxK0AQAAYESCNgAAAIxI0AYAAIARCdoAAAAwIkEbAAAARiRoAwAAwIgEbQAAABiRoA0AAAAjErQBAABgRII2AAAAjJ2r6eoAAAzESURBVEjQBgAAgBEJ2gAAADAiQRsAAABGJGgDAADAiBYctKvqjVX16nm2n1xVz5xl/T5V9bSNHPtxVfWIhdY07Ht4Vb1gU/bdElXVkVW1w4zXn66qO99G5zqwqo7ahP0uHOn8K6vqZ1X15Rk/d5in/eOq6u/HODcAAMDYViziufZJsirJp+dp87gkNyZZcIDr7ndvWllLo6oqSXX3rXM0OTLJh5PclCTdPe+XFJuju09Pcvom7LdJX4rM4Vvdvc+IxwMAAFgSU41oV9XRVfX1qvrHJA8Y1u1aVWdW1aVVdX5VPXDGLk+sqkuGfX5/GJ18U5JDhtHKQ2Y5x8okhyf5k6HNo4eRznOq6oqqOruq7j1Pjb8caa+qz1bVW6rqC0MNjx7Wb1dV/6OqrhyOecSw/glVdVlVra6q91fVHYf111bVsUM9l1TVg6vqrKr6VlUdPuPcr6mqLw7H/LN5alxZVVdV1QeTXJnkXlV14nDsNev3rar/kuTfJTm3qs6dUcvdhuVXDe/hyqo6ciPn+9owy+DrVfWRqnpiVV1QVd+oqv2Hdi+qqr8alg8ejnt5VZ03rNtjuJZfHt7jbsP6G4c/Hzdc848P5/vI8EVCquppw7pLq+qEhYxEV9X+VfX54bO5sKoeMEubx84YBb+sqnaa9jOpqsOGa3/JupvWTlsWAADAvDYatKtqvyTPzmRE+mlJHjJsOinJEd29X5JXJ3nXjN1WJtk/yQFJ3j2c5/VJTu3ufbr71A3P093XDm3fNrQ5P8k7kpzS3Xsl+UiSExbw3lZ09/6ZjAy/YVh32FDbPuuPWVXbJzk5ySHdvWcmo/wvm3Gcbw8jrecP7Z6Z5GFJ1ofiJyXZbXi/+yTZr6oeM09duyV5V3fv0d3XJTm6u1cl2SvJY6tqr+4+Ick/J/m97v69mTsPn8ehSR461PHSqtp3nvPdL8lfJHng8PPcJI/K5DP7b7O0f32SJ3f33kkOHNYdnuTtw3VYleT6WfbbN5NrvXuS+yZ55HBt35PkqUM/+bfz1LnrjMD8zmHd15I8urv3Her677Ps9+okrxhqe3SSn037mXT3Sd29qrtXbbfDzvOUBgAAML1ppo4/OsnfdPdNSVJVpyfZPskjkpw2DFwmyR1n7POxYUr0N6rq6kwC3qZ4eJJnDMsfSvLWBez7ieHPSzMJ10nyxCTv7u5fJEl3/6iq9k5yTXd/fWhzSpJXJPmfw+v1U6pXJ9mxu29IckNV3VyTe6afNPxcNrTbMZOQd94cdV3X3RfNeP2sqjosk8/iHpkE1SvmeV+PyuTz+GmSVNUnMvmMLpuj/TXdvXpouybJ2d3dVbV6xnWZ6YIkJ1fVx/Kra/j5JEdX1T2TfKK7vzHLfl/o7uuH83x5OPaNSa7u7muGNn+dyZcds5lt6vjOSU4ZRtA7ye3nqPcvq+ojQ23XD0F7IZ8JAADAaDb1Hu3bJfnJPPfU9kZeL4abhz/XZfPuRV9/nFtnLK9/vSJJJTm2u98z5fF+un6hqu6TyYjsQ7r7x1V1ciZfYoxpw5pnvp/fuC7dfXhVPTST2QiXVtV+3f3Rqrp4WPfpqvqj7j5nnvNs7jVf78+TnNvdT6/JrQWfnaXe46rqU5nMtrigqp6chX8mAAAAo5nmHu3zkvxBVf2b4f7X/5jJA7quqaqDk8mDvYaR4fUOrqrbVdWumUwjvirJDUl22si5NmxzYSbT1pPkeZlM394cn0nyR1W1Yqj7LkNtK6vqfkOb/5Tkcws45llJXlxVOw7H3KWqfmfKfX8rk+C9tqrunuSpM7bNdb3Oz+Tz2KGq7pTk6dn86/JLVbVrd1/c3a9P8v1M7iO/byYj0yck+dtMprlP46ok9x1CcpL8xr35G7Fzku8Myy+ap97V3f2WJF/MZPbE5nwmAAAAm2WjQbu7v5Tk1CSXJzkjkzCTTILvS6rq8iRrkhw0Y7dvJ/nC0P7w7v7XJOcm2b3meBja4O+SPH1o8+gkRyQ5tKquyCQAv3Khb3AD7x1qu2Ko+7lDbYdmMg1+dSYjvVM/wby7/yHJR5N8ftj/49n4Fwrr9708k+nNXxuOccGMzSclObOGh6HN2OdLmdwr/oUkFyd5b3fPNW18Uxxfk4fCXZnJFx2XJ3lWkiuHKeEPSvLBaQ7U3T9L8vJM3selmXx5sDZJqmpVVb13I4d4a5Jjq+qyzD1CfuTw8LYrktyS5IzN+UwAAAA2V3UvxaxuthVVtWN33zg8hfydSb7R3W9b6ro29LKjj+0Tj3ndUpcBAABsuWrjTSam+u+9YDO8dBgJX5PJVHD3TQMAAFu1MR5YtWBVdWh+cxr4Bd39iin2PTrJwRusPq27jxmrvs1VVXdNcvYsm57Q3T9c7udbiGH0eosbwQYAALitLEnQ7u4PJPnAJu57TJItJlTPZgi3cz2RfdmfDwAAgLmZOg4AAAAjErQBAABgRII2AAAAjEjQBgAAgBEJ2gAAADAiQRsAAABGJGgDAADAiARtAAAAGJGgDQAAACMStAEAAGBEgjYk2XOXnZe6BAAAYCshaAMAAMCIBG0AAAAYkaANAAAAIxK0AQAAYESCNgAAAIxI0AYAAIARCdoAAAAwIkEbAAAARiRoAwAAwIgEbQAAABiRoA0AAAAjErQBAABgRII2AAAAjEjQBgAAgBEJ2gAAADAiQRsAAABGJGgDAADAiARtAAAAGJGgDQAAACMStAEAAGBEgjYAAACMSNAGAACAEQnaAAAAMCJBGwAAAEYkaAMAAMCIBG0AAAAYkaANAAAAIxK0AQAAYESCNgAAAIyounupa4Al99rXvvaG29/+9lctdR1sPW688ca77bjjjj9Y6jrYeuhTjE2fYkz6E2PbQvvUD9785jc/ZZqGgjYkqapLunvVUtfB1kOfYmz6FGPTpxiT/sTYlnufMnUcAAAARiRoAwAAwIgEbZg4aakLYKujTzE2fYqx6VOMSX9ibMu6T7lHGwAAAEZkRBsAAABGJGgDAADAiARttilV9ZSquqqqvllVR82y/Y5Vdeqw/eKqWrn4VbKcTNGnXlVVX6mqK6rq7Kr63aWok+VjY31qRrs/rKquqmX7X59w25umP1XVs4bfU2uq6qOLXSPLyxR/7927qs6tqsuGv/uethR1sjxU1fur6ntVdeUc26uqThj62xVV9eDFrnFTCdpsM6pquyTvTPLUJLsneU5V7b5Bs5ck+XF33y/J25K8ZXGrZDmZsk9dlmRVd++V5ONJ3rq4VbKcTNmnUlU7JXllkosXt0KWk2n6U1XtluR1SR7Z3XskOXLRC2XZmPJ31J8m+Vh375vk2UnetbhVssycnOQp82x/apLdhp/Dkpy4CDWNQtBmW7J/km9299Xd/fMk/zvJQRu0OSjJKcPyx5M8oapqEWtkedlon+ruc7v7puHlRUnuucg1srxM83sqSf48ky8C/3Uxi2PZmaY/vTTJO7v7x0nS3d9b5BpZXqbpU53kt4blnZP88yLWxzLT3ecl+dE8TQ5K8sGeuCjJnavqHotT3eYRtNmW7JLkn2a8vn5YN2ub7v5FkrVJ7roo1bEcTdOnZnpJkjNu04pY7jbap4Zpc/fq7k8tZmEsS9P8jrp/kvtX1QVVdVFVzTeyBNP0qTcmeX5VXZ/k00mOWJzS2Eot9N9aW4wVS10AwLagqp6fZFWSxy51LSxfVXW7JH+Z5EVLXApbjxWZTMl8XCYzbs6rqj27+ydLWhXL2XOSnNzdf1FVD0/yoap6UHffutSFwWIyos225DtJ7jXj9T2HdbO2qaoVmUx5+uGiVMdyNE2fSlU9McnRSQ7s7psXqTaWp431qZ2SPCjJZ6vq2iQPS3K6B6Ixh2l+R12f5PTuvqW7r0ny9UyCN8xmmj71kiQfS5Lu/nyS7ZPcbVGqY2s01b+1tkSCNtuSLybZraruU1V3yOQBHadv0Ob0JC8clp+Z5Jzu7kWskeVlo32qqvZN8p5MQrZ7H9mYeftUd6/t7rt198ruXpnJff8HdvclS1MuW7hp/t77ZCaj2amqu2UylfzqxSySZWWaPvXtJE9Ikqr695kE7e8vapVsTU5P8oLh6eMPS7K2u7+71EVNw9Rxthnd/Yuq+uMkZyXZLsn7u3tNVb0pySXdfXqS92UyxembmTyY4dlLVzFbuin71PFJdkxy2vBcvW9394FLVjRbtCn7FExlyv50VpInVdVXkqxL8pruNpOLWU3Zp/5rkv9VVX+SyYPRXmTQgrlU1V9n8mXf3Yb7+t+Q5PZJ0t3vzuQ+/6cl+WaSm5IcujSVLlzp9wAAADAeU8cBAABgRII2AAAAjEjQBgAAgBEJ2gAAADAiQRsAAABGJGgDAADAiARtAAAAGNH/B7T4BtFeDDiwAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1008x720 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# GBM var importance\n",
    "gbm_default.varimp_plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Shapley Importance for MGBM and GBM"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Shapley for MGBM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "First 5 rows of contributions:\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead>\n",
       "<tr><th style=\"text-align: right;\">    term_360</th><th style=\"text-align: right;\">  conforming</th><th style=\"text-align: right;\">  loan_amount_std</th><th style=\"text-align: right;\">  loan_to_value_ratio_std</th><th style=\"text-align: right;\">  no_intro_rate_period_std</th><th style=\"text-align: right;\">  intro_rate_period_std</th><th style=\"text-align: right;\">  property_value_std</th><th style=\"text-align: right;\">  income_std</th><th style=\"text-align: right;\">  debt_to_income_ratio_std</th><th style=\"text-align: right;\">  debt_to_income_ratio_missing.False</th><th style=\"text-align: right;\">  debt_to_income_ratio_missing.True</th><th style=\"text-align: right;\">  debt_to_income_ratio_missing.missing(NA)</th><th style=\"text-align: right;\">  BiasTerm</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td style=\"text-align: right;\">-4.57106e-06</td><td style=\"text-align: right;\">  0.00735352</td><td style=\"text-align: right;\">        0.0514138</td><td style=\"text-align: right;\">                 0.842012</td><td style=\"text-align: right;\">               -0.00580597</td><td style=\"text-align: right;\">           -0.00170303 </td><td style=\"text-align: right;\">            0.553635</td><td style=\"text-align: right;\"> 0.0124719  </td><td style=\"text-align: right;\">                 0.267141 </td><td style=\"text-align: right;\">                        -0.000184935</td><td style=\"text-align: right;\">                                  0</td><td style=\"text-align: right;\">                                         0</td><td style=\"text-align: right;\">  -2.87949</td></tr>\n",
       "<tr><td style=\"text-align: right;\">-1.50614e-06</td><td style=\"text-align: right;\">  0.00768744</td><td style=\"text-align: right;\">       -0.073352 </td><td style=\"text-align: right;\">                 0.573904</td><td style=\"text-align: right;\">               -0.0085259 </td><td style=\"text-align: right;\">            0.00141006 </td><td style=\"text-align: right;\">           -0.104249</td><td style=\"text-align: right;\">-0.00768904 </td><td style=\"text-align: right;\">                -0.290046 </td><td style=\"text-align: right;\">                        -0.000237618</td><td style=\"text-align: right;\">                                  0</td><td style=\"text-align: right;\">                                         0</td><td style=\"text-align: right;\">  -2.87949</td></tr>\n",
       "<tr><td style=\"text-align: right;\">-4.57106e-06</td><td style=\"text-align: right;\">  0.00822359</td><td style=\"text-align: right;\">        0.370754 </td><td style=\"text-align: right;\">                 0.596289</td><td style=\"text-align: right;\">               -0.0673389 </td><td style=\"text-align: right;\">           -0.615974   </td><td style=\"text-align: right;\">            0.283736</td><td style=\"text-align: right;\">-0.000276035</td><td style=\"text-align: right;\">                 0.0216484</td><td style=\"text-align: right;\">                        -0.000126609</td><td style=\"text-align: right;\">                                  0</td><td style=\"text-align: right;\">                                         0</td><td style=\"text-align: right;\">  -2.87949</td></tr>\n",
       "<tr><td style=\"text-align: right;\">-1.50614e-06</td><td style=\"text-align: right;\">  0.0116721 </td><td style=\"text-align: right;\">       -0.0421028</td><td style=\"text-align: right;\">                -1.15109 </td><td style=\"text-align: right;\">               -0.0262746 </td><td style=\"text-align: right;\">           -0.0309957  </td><td style=\"text-align: right;\">           -0.411278</td><td style=\"text-align: right;\">-0.0237093  </td><td style=\"text-align: right;\">                -0.1714   </td><td style=\"text-align: right;\">                        -0.000212137</td><td style=\"text-align: right;\">                                  0</td><td style=\"text-align: right;\">                                         0</td><td style=\"text-align: right;\">  -2.87949</td></tr>\n",
       "<tr><td style=\"text-align: right;\">-4.57106e-06</td><td style=\"text-align: right;\">  0.00469299</td><td style=\"text-align: right;\">        0.137133 </td><td style=\"text-align: right;\">                 1.05481 </td><td style=\"text-align: right;\">               -0.00406382</td><td style=\"text-align: right;\">           -0.000802108</td><td style=\"text-align: right;\">            0.341723</td><td style=\"text-align: right;\"> 0.0127726  </td><td style=\"text-align: right;\">                 0.252466 </td><td style=\"text-align: right;\">                        -0.000191413</td><td style=\"text-align: right;\">                                  0</td><td style=\"text-align: right;\">                                         0</td><td style=\"text-align: right;\">  -2.87949</td></tr>\n",
       "</tbody>\n",
       "</table>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "shap_values_mgbm_default, expected_value_mgbm_default = get_shapley(mgbm_default, train, X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAApwAAAFeCAYAAAAog8+HAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xmc3uO9//HXxFIliTUoIgvKT/f23aKtpTg5xzJ1Sp1YIzQOKkUVUWsslRIaVVQsFRL7UgSxtaJxFP3Q0ipVZJMmBIlm0ZJkfn9c183XbZZ7xnwzk8n7+XjMY+77u1zX9f3eE95zLd+pa2howMzMzMysLN06ugFmZmZm1rU5cJqZmZlZqRw4zczMzKxUDpxmZmZmVioHTjMzMzMrlQOnmZmZmZXKgdPMzMzMSuXAaWZmZmalcuA0MzMzs1Kt2NENMOtKxo8f31BfX9/RzTAzM1ta6mo5yD2cZmZmZlYqB04zMzMzK5UDp5mZmZmVyoHTzMzMzErlwGlmZmZmpXLgNDMzM7NSOXCamZmZWakcOM3MzMysVA6cZmZmZlYqB04zMzMzK5UDp5mZmZmVyoHTzMzMzErlwGlmZmZmpXLgNDMzM7NSOXCamZmZWakcOM3MzMysVHUNDQ0d3QazLqPu/EX+B2VmZp1Sw3ErllFsXS0HuYfTzMzMzErlwGlmZmZmpXLgNDMzM7NSOXCamZmZWakcOM3MzMysVA6cZmZmZlYqB84OJGmipFM6uh3WMkkvSRrc0e0wMzNbFjlw2jJH0g6SFrVTWQ79ZmZmJXPgNDMzM7NSlfLIeWs9SZ8HLgS+BMwBfgWMiIjFef/VwM7AGsB04OyIuD7v2wF4CNgfOAdYB7gf+F5EzGuh3i8AFwGfAVYAHgeGRsTLef+YvP09YE9gAXAc8DxwBbAFEMD+EfGPfM7awChgQK7mfuCHEfFW3j8FOCUixuX3fYHJQO+IeLVQ57+AvXOdZ0bEaEkbABOAFSTNz+UfGRHXNHONOwMjgU2Ad4E/RcTOki4GtgW2kXQiMCMiNpe0EnAucACwJF+LmZmZtZF7ODsBSasDDwIPA+sDuwGHAMcWDnsU+CIpcJ4JjJG0ZWH/CqSA9wXg06TgelQN1TcAw4ENgb7AfGBc1THfBW4D1gLOIgXNM4HvAOvlMs4oHH8dsCbw//LXOsDYGtpSXef4XOcPgIsl9cmhdhdgcUR0z19Nhs3sWlKoXj1f59kAETEUmASclcvZPB9/IrA78HWgH+m+9Gll+83MzCxzD2fnsBup5+3siGgAnpd0LilwjgSIiKsKx98o6ThgB+Cvhe0nRsR8YL6kOwC1VHFEPFt4+29JZwB/lrRqRCzM238bEfcASLoW+CUwNiJezdtuBQ7NrzcA/hP4dETMyduOBV6Q9KmImFnjPfltRNyVX98uaS4pcE+t8fyid0m9m+tFxCxgYgvHDwJ+GhEvAeR7/b021GtmZmY4cHYWvYGpOWxWvJy3I6kbqRdyIKkHtAFYDehVOH5xRMwuvF8A9GipYkmbkELtVvn4Sht68UG4ez8kRsRCSR/aBiws1NU7f59cdS2VfbUGzurjarqeJuwBnEQK0rOByyPiwmaO3wiYUnkTEQskvd7Gus3MzJZ7HlLvHKYDfSTVFbb1z9sB9gWGAHsBa0bEGsAzQB0f32XAPODzEdET+Ebe3tayK23uW9jWv2rfPFJgrtiglXUsac3BEfFMRAwE1gUOA0ZI2rGZsmZQaL+k6nBvZmZmreDA2TncA3wCOEnSypI2B4YBlWH0nsAiYDbQTdIhpLma7aEnqfdwrqR1SHMz2yzPsXwAuEDSGpLWBC4AJhSG058C9pXUXVIv4NRWVjOLtGioX0sH5vt5kKR1cg/yHFLIXFwoa9Oq08YCx0vaRNIngfPwvxUzM7M28/9EO4GIeJu04Gdn4DXSqu5rgZ/lQ64BngBeIvW+bUla7NIefkhaqf3PXObd7VDmAaRezL8BLwBzSfMiK04hBb6ZpPmUN7am8Ih4kTSP9ElJcyUd2MIpA0lzSOcDdwGnR8Qjed8oQLmc5/K2EaTP4HHS1IBptG3uqJmZmQF1DQ0NLR9lZjWpO3+R/0GZmVmn1HBcKUt3apqC5x5OMzMzMyuVV6l3cZK2JT0ovTHnRMQ5S7M9ZZC0PzC6id2HRcR1S7M9ZmZm9mEeUjdrR+PHj2+or6/v6GaYmZktLR5SNzMzM7OO58BpZmZmZqVy4DQzMzOzUjlwmpmZmVmpHDjNzMzMrFQOnGZmZmZWKgdOMzMzMyuVn8Np1o78py2tTCX9WTozs4/Dz+E0MzMzs47nwGlmZmZmpXLgNDMzM7NSOXCamZmZWakcOAskTZB0Qke3Y3nwce+1pAMkTWnHJjVX15WSxiyNuszMzLoiL3ksiIhdaj1W0kTgoYg4u7wWtZ6kwcApEbFpR7elOa2512WRNBz4ZkTs3NFtMTMz68rcw1kiSSt15vI6Qle4BjMzM2sd93AWVHotgXHAZGAQ8GOgN/B74KCImCnpYmBbYBtJJwIzImLzPOy6EvAe8G3gJuAISdsD5wFbADOBURExusb2/AnoC+wInCNpHHAl8BVgZeBZ4JiIeErSNsBlwMqS5udido+IiZI+C1wAfBl4B7gOOC0i3muhDZVrWgLsAcwGzoqIMYVjtgVGAFsCc4BLgZ9FRIOkHfI9PRg4A+gF9KjuIZb0eeBC4Eu5jF8BIyJicd7/tVzuFvmePNDS/cvn9QVGA1sBDaTPdV/gi8BJQLfCvfp8RLwi6RDg5NzWO0nPGFtUS31mZmb2Ue7hbN5AYDtgQ2A14EyAiBgKTCIFr+4RsXnhnL2BCaSw8iNJ/YD7gF8CawODgRGS9q6xDYcAFwGr5+/dSMGrD7A+8DRwu6SVIuL3wOHAK7ld3XPYXBd4BLg9X8s2wH+QwnQt/ge4H1gLOAz4paSvA0jaErgXGJmveTdgKHBg4fwVgF1JYXK96sIlrQ48CDycr2m3fN3HFvZPAG7Nbfgh8P0a234OMC3Xuw7p/s+JiJvyvomFe/VKDs+XkO7jWrldA2usy8zMzBrhHs7mnRERbwBIuh4YUsM5j+YwA7BQ0r7A04Uewccljc5l3VJDebdGxG8r5ZHC07TKTkmnAEcBmwF/baKMQcAzhV7VGZJGAOeSQ3QLHo+Icfn1Q5JuIwW3x0jB75aIuDPvfyH3AA8Cri2UMSwi3m6i/N2Ad4GzI6IBeF7SuaTAORLYHVgAnJv3/0HSVcD+NbT9XVKI7R8Rz5N6hJsziHTPH8zvr5V0WA31mJmZWRMcOJs3s/B6AdCjhnOmVL3vTRrGLXqZNDxdiw+VJ2kd4GfADsAapKFuSL2LTekHfEPS3MK2OlLPY6vbkN9/uVD2jpL2LOzvBkwvvF9S9b5ab2BqDpMVL+ftABs1sr/6njbleOBUYLyk1Ui9pD+OiPlNHL8REFXbaq3LzMzMGuHA2XZLatw+nTScXNSf5gNYc+WNAD4FbJXnk/YA/skHf8u0sXZNJc2X3K3GOqv1beT9q4WyfxURRzZzfkNVWKw2Hegjqa5wXPEezWhkf3WbGhURs0k9wEdJ6k+ak3kCcBqN36sZjZTdF3iplvrMzMzsoxw4224WUMujh24ATpU0CLie1DN4GHBEG+vtSRpanyOpO2lYvLpd60rqGRH/zNuuJc0nPSS34V1SiPp0RNxXQ51b56kBNwPbA3sBlUcJXQo8Iuk+0lzVBuDTQK+IeKTGa7qHtGDoJEkjSb2mw0iLfQDuJs1fPV7SKOBzwPeAf7dUsKSBwJOkXtm3Sde+OO+eBWwsaeWIeDdvGwvclxdLPQLsQ1pw5MBpZmbWRl401HajAEmaK+m5pg6KiMmkHs6hwJukQHNqRNzcxnpPA9bNZT1Lmke5uLD/YdJCl8m5bdtHxCzgW8B/k4LXHODXpF7EWtycr2EOcBVwZET8X76+v5DmWB5DmoLwOjCG5of4PyTP7RxACrGvkRYoXUuaOkBEzCXN8xyY23ARaRFWLb5ECo7zgedIi6xG5n23kHpRZ+V71S+H5B+QngTwFvBfpKcNmJmZWRvVNTQ0N9Jpy7vc07coImpZMLXcqzt/kf9BWWkajvOglJl1OnUtH+IeTjMzMzMrmX9d7kCSTiI9fLwxu0TEpKXQhv35YK5ktU7/OCBJG9P046DGRcThS7M9ZmZm9lEeUjdrRx5StzJ5SN3MOqGahtT9Xy+zdnTX5hOor6/v6GaYmZl1Kp7DaWZmZmalcuA0MzMzs1I5cJqZmZlZqRw4zczMzKxUDpxmZmZmVioHTjMzMzMrlQOnmZmZmZXKD343a0d+8HvX4getm5m1yH9L3czMzMw6ngOnmZmZmZXKgdPMzMzMSuXAaWZmZmalcuA0MzMzs1I5cNr7JE2QdEJHt6OzkXSlpDEd3Q4zM7NllZ/5Ye+LiF06ug1Lk6ThwDcjYueObouZmVlX5h5OMzMzMyuVezjtfZImAg8B44DJwCDgx0Bv4PfAQRExMx/bHRgO7An0AqYDh0XEJEmrAiPyvk8CjwJHRcS0Qj1PA/2AnYHXgf8lPTz2QmBj4DfAoIiYl89ZGzgPGACsAjwM/CAiXmvhmvoCo4GtgIZ8XfsCXwROArpJmp8P/3xEvCLpEODkfF135nYtasWtNDMzswL3cFpzBgLbARsCqwFnFvZdRQpxOwE9gW8DM/O+UcDW+asP8AYwXtIKhfMPBH4KrAHcBIwlhc7tgL7A5sBRAJLqgDtIgfGzucx5wPU1XMM5wDRgPWAdYDAwJyJuyvsmRkT3/PWKpG2BS4DDgbWAB/N9MDMzszZyD6c154yIeANA0vXAkPx6XeB/gM9GxOR87Et5XzfgIKA+ImbkbccAbwFfI/WUAtwcEU/k/eNIPakjI+KtvO1uQPnYr+SvnSPi33n/CcAbkjaKiFebuYZ3gfWB/hHxPPBsC9c8CLg1Ih7M76+VdFgL55iZmVkz3MNpzZlZeL0A6JFf983fX2zknF7AJ0hD1wBExHzSsHnvJspe2MS2Sn39cpmvSZoraS7wMvAv0vB7c47PbRkvaaakX+TpAE3ZCJhStW1yI8eZmZlZjdzDaW0xJX/fDPhr1b7ZwL9JobTS69kdWJc0z7MtppIC71oRsaQ1J0bEbNLQ/FGS+pPmZJ4AnAY0VtYMPgjUFX3J12JmZmat58BprRYRr0u6FbhU0mBSINwk73tJ0rXAWZL+CswFLgBeAJ5sa5XAM8BFkk6PiDcl9QJ2iogbmztR0sBc7xTgbdIQ++K8exawsaSVI+LdvG0scF9+7uYjwD6kuaoOnGZmZm3kIXVrq0OAP5FC2TxSz+H6ed8PSSHxD6QFO58Cvh0Rixspp0W5V3MP0mrxpyTNAx4Hdqjh9C/lNs4HniOtjh+Z991C6nWdlYfq+0XEI8APgCtJ807/i7SoyczMzNqorqGhoaPbYNZl1J2/yP+gupCG4zwIZGbWgrpaDnIPp5mZmZmVyr++2zJN0sZ8dOFSxbiIOHxptsfMzMw+ykPqZu1o/PjxDfX19R3dDDMzs6XFQ+pmZmZm1vEcOM3MzMysVA6cZmZmZlYqB04zMzMzK5UDp5mZmZmVyoHTzMzMzErlwGlmZmZmpfJzOM3akf+0ZeP8JyLNzLosP4fTzMzMzDqeA6eZmZmZlcqB08zMzMxK5cBpZmZmZqVqU+CUtJGkBkl9azh2uKSH2lLP0iDpMkkXd3Q7OrOOvEftUbek/SU9015tMjMzs9bp8KWjObROBnpHxKs1HD8YOCUiNm2P+iPi8PYop6uQNIV0f8dVtnXkPWqPuiPiOuC6dmiOmZmZtYGH1JcTklbq6DaYmZnZ8qmmHk5J6wOXA9sDrwHnVe0/FDga6A28AgyLiAcKh9RJGgUMAt4BLo6In+Z9laHOv0lqAM6NiLOaaMc2wGXAypLm5827R8RESdvndm0BzARGRcToGq5tDLAoIobk9w3AkcDBuazngMER8ULevxJwPHAQsAHwer7eWyWtCJwEDAbWBJ4Gjo6IvxTqWgF4D9gTWAAcBzwPXJHrC2D/iPhHPmdV4ExgL2B14ElgaES81MJ17QA8lK/jDKAX0EPS0cARwIbAHFLP3ykRsVjSeGBj4EpJlwGPRcSARu5RH+Ai4Bukz/M24McR8U4LbRoObJuv8RDSLzw/yedfDXwVeBE4ICKeL9yzRRExRFIdcHa+ph7Am8AFEfELSWuSfkZ3JP1cvwocHhGTqnvFJU0EngL6AgNIn+GxEXFn3l8H/Bj4PrAqcA3weWBSRAxv7hrNzMzso2rt4bwOWEwKI9uRAhXwftgcBuxPClknA7dLKg55b0cKqp8C9gCOlbRf3veF/H3ziOjeVNgEiIjfA4cDr+Rju+ew2Q+4D/glsHZu3whJe9d4fdUGkwLeOsB04BeFfWcDBwB7Az1JIfzFvO94UqjeFVgfmAQ8KKln4fzvkgLWWsBZpKB5JvAdYD2ggRQQKypBdOtc5hPA3TX2WK6Q2/KlXDakILZLbvsepOA3BCAi6oFpwJB8bwdUF5hD9T3ALKBPbtc3gPNraA+kn4W/52s5ABgJXEUK+WuRwvdFTZz7H6Sgv1VE9AC+Bjya9x1PCod9gDVI97O5KRoHAReQQvzFwDU53AMcSPoFqp5032bmdpuZmVkbtNjDKWlDUq/RphHxNvC2pDOASg/m0cCZEVHpqbxX0sPAPqRwBul/2OdGRAPwlKTLSaHu+na6jn2BpyNiTH7/uKTRpCB1SxvKGxkR0+D9HrZx+XUdKRgNjIhn87Gv8kGwOZh0nZXe0DNzG3YDbsjH/DYi7sn7ryWF5LGV+auSbgUOza/XAfYD+kTEa3nbGcAxwFZ8ELaaMyx/bgBExG2FfX+UNBbYCWixNzj7GrAZKfQtABZIOgW4Q9LQ/Bk358WIuDK/niDpTeD+Qo/m9TQ93/JdYBXgM5JmR8TrpN7Jyr61gc2BP0bEi02UUXFTRDyW67wc+Fm+rmdIvzSMjog/5v0jSZ+7mZmZtUEtPZwb5e9TC9smF173Ay6RNLfyBXyLNGRbMbUqiEwplNseele1CeDlvL0tZhZeLyAN30Iall6ND3o0m21HRCwhXWuxHTML+xc2Ut/CQn398vdnC/f2LWAlaru2JaQe2vdJ2lfSHyS9KeltUpDqVUNZFb2B2TlsVrxMCoK1lDOz6v1Cmr7+D4mIiaQpC6cAr0t6QJLy7pHAb0jD37MlXSNpvcbKqW5H4Voq9W5I4ec9/+x+6D6amZlZ7WqZwzkjf+9DChaQ5r5VTAVOj4jmehL7SKorhM6+fNAruKS2pr6vseOnk4aOi/rT/iFhNikQbUYaFm6sHX0rbyR1y+/b2o5K6NksIma34fyGYtCX1JvUW7snMCEi3pV0PqDCOS19HtOBXpJWLQTm/sC/SPenVBFxOXB5Hv4eDtwObJxD48nAyXnO8ThSCB3UhmpmkH7egfd7ttv6y4uZmdlyr8XAGRGv5kUW50k6GPgkcFrhkFHAcEl/Jw1HrgJ8BXijMrRMmrt5fF449FnSkPGxed9sUsjZjObn3FXMAtaV1DMi/pm33QCcKmkQaZj+y8BhpMUx7SYiGiRdSroX00gLijYE1spD7GOAEyT9jtSzOYx0j+9pY32v5yHmSyUdExEzJK1B6kF+MCLmt1BEte6kXu3ZwHuStibNV3y+cMws0mfRlCeBl4ALJP2INF/yLODqGobTPxZJXwM+kdvwb2AeaW4xkupzu14E5pMC8OI2VjUWOFfS7aTP+CjSAjEzMzNrg1oXDe1H+h/9dNJCmGsrOyLiCtLq8KtJq56nAaeShn0rJpFC5yzgbuDn5PmbeWXzqcANedj45Bba8jDwIDA5H799REwm9XAOJa1cHgucGhE313h9rXEycDNwBynwTAQqC6RGksLvA6RFUjsCAwrBuC0OBf4GTJQ0D/gzacFSq8Ndnid5OnAnMBc4kQ/mllacDRwgaY6kCY2UsQjYnTQlYhop/D1BWm1ftu6kn503SJ/zAGBg3rcJMB74Jynsv0MK/G1xLXAJcC/pc9wIeJwUcs3MzKyV6hoaSu2UMlvm5akR04HjI6LZhW515y/yP6hGNBzX4X9jwszMylFXy0H+v4BZIyTtQ+rF7kZ6JueqwEd6fM3MzKxlnS5wStqWpv/Hfk5EnNPK8k4irWxuzC4RMak15XUmhYffV5sUEbss1cbQ/p9dBxvKB4+K+guwa0TM6cD2mJmZLbM8pG7Wjjyk3jgPqZuZdVkeUjdb2u7afAL19fUd3QwzM7NOpdZV6mZmZmZmbeLAaWZmZmalcuA0MzMzs1I5cJqZmZlZqRw4zczMzKxUDpxmZmZmVioHTjMzMzMrlR/8btaOynjwux+abmZmnVhND353D6eZmZmZlcqB08zMzMxK5cBpZmZmZqVy4DQzMzOzUjlwmpmZmVmpHDgzSfMlbdPR7VhefNz7LelKSWPasUnN1fWSpMFLoy4zM7OuyM9bySKie2uOl9QAbBsRj5bUpDbJIWxRRAzp6LY0p7X3uwySJgIPRcTZHd0WMzOzrsw9nCWStFI7llUnaZn/BaE974mZmZktG5a5ACNpCnA5sBOwFTAF+N+IeCwHspOAwcCawNPA0RHxlxrKfb/HMg+fngJcBJwArAbcDHw/IhZLeiaf9oCkJcCNETEkt+1XwLeArwJDgBslHQEcA6wPPA8cHxGTamzTMcCBwGeAb0laDTgH+DSwCPgNcFREvC7pBGD/fO4+uZjVc5v/GzgV2ASYCZwdEdfV0IbKNQ0Avgi8ABwREX8oHHMocDTQG3gFGBYRD+R9w4HtSJ/Fgfn7LtU9xJL2Ak4D+pI+0+ER8etCHYcAJwO9gDtJD5pdVEP7dwZG5ut+F/hTROws6WJgW2AbSScCMyJi8xyIzwUOAJYAo1qqw8zMzJq3rPZwHgIcBawOPAhck7cfDwwCdiWFu0nAg5J6tqGOPsB6pKDyVWBvYB+AiPhCPmZARHSvGr4+FDgW6AHcKWlf4KzcrrWBK4D7JPWpsR3fAwYC3YE/Av8GhpKC1+eADYCf53adB1wHXJPb1T2Hzf8AriKF17WAg4CLJW1XYxsOJwXKtYBbgXsr9zSHzWGkoLsmKRTeLmnTwvnbkUJub2Cv6sIlfT23+0TSPToJuEHSVnn/tsAluR1rkT7zgTW2/VrSLw6rAxsCZwNExFDSz8dZ+T5tno8/Edgd+DrQjxSAa/2szMzMrBHLXA9nNjoinoO0eAQ4RtLqwMHAuRHxQt53JqmXcTfghlbW8Q5wWkQsBl6S9BtApGDUnCsi4o+VMiQdnNv7RN52laQhwH7AiBracX5EvJxfLwaKc0ZnSTqP1APZnKOBnxd6VZ+UNI4Ugn9XQxuuioinACSdC3yfFMquz2WfGRGVXt97JT1MCueVuZFTI+KC/PrdRsofDNwWERPy+3sk/Zr0i8UTuZ23RsSDef+1kg6rod2V+jYB1ouIWcDEFo4fBPw0Il7K13scKfSbmZlZGy2rgXNm4fWC/L0HqQdtcmVHRCzJQ8K921DH6zlsFuvpUcN5U6re9yYNxxe93Io2fag8SV8hDal/AViVNLTc0gKcfqTh+GML21Yg9fC1qg0R0SBpGrBRoexLJF1UOH5F4NXC+6ktlN8beKpq28vAl/PrjYCo2j+Z2uxB6jH9s6TZwOURcWEzx2/Eh693gaTXa6zLzMzMGrGsBs6mTCcNgQIgqVt+P72Euhqa2L6kuTZl/YHxNdZTXd6NpGHtvSPin5J2ryqr+nhIgW9MRIyssc5qfSsvJNUBG/NBoJwKnB4RtzRzfmNtKmrqHlU+txmN7O8LvNRCueSe14G53d8kzbt9NiJ+20S7PlRXnjPbq6V6zMzMrGldLXCOAU6Q9DtSL9Uw0jXeU0Jds4DN+PAQd1Nt+rmku/hg4cwXgX3bWG9P4G1gnqSNSXMOq9u1taRuEVEJVBcCYyQ9DjxG6t38HFAXEdU9h405JA9x/xn4IalntXJPRwHDJf0deAZYBfgK8EZlakMNrgEekjQWeIi0QGlPYIe8fyxp3usY4BHScP1WtBA4Ja1Mus/3RMQbkuaQQmal53oWsGnVaWOB4/Mjk/4BnMeyO9fZzMysU+hq/yMdSZqr+QDwGrAjaWHPP0uo62TgTElzJI1u6qCIuB44AxgHvAkcAewaES0NMzflf0nzUucBtwPVPYtXklbVvylprqQV8orxQ0n35w3SlIRRtDwUX3E5aeHNHNJind0i4u18fVeQQtnVef800mr4mh9/FBH/R1rIdH4u4zzggIh4PO9/BPhBvra3gP8Cbqqx+IHAC5LmA3eRemMfyftGAcr36bm8bQRwP/A4adh+Gi1PCTAzM7Nm1DU0NDUybPb+Y5FOiYhxHd2WZUHd+Yva/R9Uw3FdbSDCzMy6kLpaDupqPZxmZmZm1sksN10neci0secpTo2Izyzt9gBImkB6+PhHLK0//SjpMtJDzhuz5dJow8chaX+gqSkNh9XycHszMzMrl4fUzdrR+PHjG+rr6zu6GWZmZkuLh9TNzMzMrOM5cJqZmZlZqRw4zczMzKxUDpxmZmZmVioHTjMzMzMrlQOnmZmZmZXKgdPMzMzMSuXncJq1o5b+tKX/TKWZmXUxfg6nmZmZmXU8B04zMzMzK5UDp5mZmZmVyoHTzMzMzErlwGlmZmZmpXLgNDMzM7NSdcnAKWmipFM6uh2diaQGSd/s6Ha0hqTnJA3s6HYASFokaYeOboeZmdmyyA8FLImkwcApEbFpR7els5PUF5gM9I6IVyvbI+IzS6HuKaTPaVzZdZmZmS2vumQPp3Ueklbq6DaYmZlZx+ryPZySPg9cCHwJmAP8ChgREYvz/quBnYE1gOnA2RFxfd63A/AQsD9wDrAOcD/wvYiY10yd2wCXAStLmp837x4REyVtD5wHbAHMBEZFxOgWrmE34Gpgw4h4L2/rDswCdouIRySdA+wDrAu8BvwiIi5sorzBVPW+ShoDLIqIIfn9xsDPgG8CDcB44EfNXXc+bziwHfA0cGD+vktz9xl4Jn//m6QG4NyIOKu697Et9y6ftw9wOrARsBC4LyIOkjQe2Bi4UtJlwGMn1SKqAAAgAElEQVQRMUBSD+BioB6YB5zWUh1mZmbWtC7dwylpdeBB4GFgfWA34BDg2MJhjwJfJAWhM4ExkrYs7F8BGAB8Afg0Kbge1Vy9EfF74HDglYjonr8mSuoH3Af8ElgbGAyMkLR3C5dyH7Aot79ib1Lg/F1+/1dSOOwBHJrL/c8Wym2UpFWA3+Yy+wFbksLaz2ssYjtSIOwN7JW3NXefv5C/b57v1VmNtKlN907SqsBY4MiI6AH0B64EiIh6YBowJNc7IJ92IbBZvu7PA3uQfg7MzMysDbp6D+duwLuk3rQG4HlJ55IC50iAiLiqcPyNko4DdiCFrYoTI2I+MF/SHYDa2J59gacjYkx+/7ik0cAQ4JamToqIxZLGAgcDd+TNBwNX5+uiag7ibyXdA+xE6pFtrd2Buoio9Oy9I+lU4DFJh1Z6h5sxNSIuyK/fze2r5T43p033LnsP2ELSnyLiLWBSUwdK6kbq0d4tImblbcOA79TYTjMzM6vS1QNnb1L4aShsezlvr4SL4cBAUg9oA7Aa0Ktw/OKImF14v4DUi9jW9kyu2vYyqQetJVcDz0paN9f/dWC/yk5JR5F6NjcC6oBPAtc3Uk4t+gEbS5pbtb2BdJ9mtHD+1OKbGu9zS9p07yJioaRdSb9k/ETSK8AFheH8ar2ATwBTCtuq6zUzM7NW6NJD6qS5gn0k1RW29c/bIfWaDSEN+64ZEWuQ5hPW8fEtaaI9fau2FdvTpIh4AXgKOIA0nPxQZUW3pG8A5wKHAevk6xhP09cxjxT4ijYovJ4KvBgRa1R9rRIRLYVN+Oi1t3SfG7tX1T7OvZsYEd8mzcE9GxgnaZMm6n6D1CtbrKu6XjMzM2uFrt7DeQ9pPt5JkkaSeu6GAZWFJj1JcyNnA93yYpovAHe3Q92zgHUl9YyIf+ZtNwCnShpE6n38MikkHlFjmVeT5o/2AI4vbO8JLCZdR0NeZLQLTQ81/ym3bXfgXlIv4XZAZVj+blJv4EnAL4D5pED6tYj4dY1tLWrpPs8mBb/NgFcbK4A23jtJ65Hmtj4UEW8Xem0r0wJm5XqB96cvXA+cIekvwDvAT2u/VDMzM6vWpXs4I+Jt0oKfnUkrt+8HriWtvga4BngCeIk0TLwlzczva6WHSQuWJkuaK2n7iJgM7AoMBd4kLWY5NSJurrHMG0m9et2BOwvbK9f1JKmH7rtAk8EwIl4GjgYuB94C/gu4rbB/IbAj6X68ALwN/Ia06Kctmr3PEfEOcCpwQ75XJzfS5rbeu27AkcAUSfOAS4CDImJK3n82cICkOZIm5G1Hk4bRXwD+TOotbmneqpmZmTWhrqGhoeWjzKwmdecvavYfVMNxXX1QwczMljM1TUPs0j2cZmZmZtbx3N3SRpK2BSY0sfuciDinleWdBJzUxO5dIqK9hvo/lva+7vbQme7dXZtPoL6+fmlVZ2ZmtkzwkLpZOxo/fnyDA6eZmS1HPKRuZmZmZh3PgdPMzMzMSuXAaWZmZmalcuA0MzMzs1I5cJqZmZlZqRw4zczMzKxUDpxmZmZmVio/h9OsHTX3py39Zy3NzKwL8nM4zczMzKzjOXCamZmZWakcOM3MzMysVA6cZmZmZlYqB07rEJJ2kLSoo9tRK0mLJO3Q0e0wMzNbFjlwLkckDZb0Uke3ozORNEXSAR3dDjMzs67MgXMpk7TS8lSvmZmZmR8M2A4kTQF+BQwAvgi8ABwREX+QNAZYCXgP+DZwE3CEpO2B84AtgJnAqIgYncvbAXgI+B5wJtADuAsYGhHz8zFr5/MHAKsADwM/iIjXqtr0LeCrwE+A04GVJc3PTd8dOBe4MSJGFa7nTOAbEbFTM9c8EtgsIv67sG0HYDywPtAAjAO+DqwKvAQMi4gHmyhvDLAoIoZU3ddTImJcfr8tMALYEpgDXAr8LCKafZispH3ytW8ELATui4iDJI0HNgaulHQZ8FhEDJDUA7gYqAfmAac1V76ZmZk1zz2c7edw4GhgLeBW4F5JPfO+vYEJQC/gR5L6AfcBvwTWBgYDIyTtXShvBVLg+Tzw/4BPAz8DkFQH3EEKdZ8F+pCC0fVVbToUOJYUWC/MbXwlIrrnr4nAaFKwJZfdDTgYuKKF670a2FVSr8K2g4GbI2IB6WfrdmCzfI03ALdVHV8zSVsC9wIjSfdxN2AocGAL560KjAWOjIgeQH/gSoCIqAemAUPy/RiQT7swt3tL0v3fg/R5mJmZWRu4h7P9XBURTwFIOhf4PqkHEeDRiLgpv14oaV/g6YgYk7c9Lmk0MAS4pVDmsIh4G3hb0mnA3ZIOB74MfAXYOSL+nes8AXhD0kYR8Wo+/4qI+GN+/Y6kxtp9IzBK0tYR8Tjwn8AngV83d7ER8VdJfwQOyOf3AL5L6nEl98SOK5wyUtIwUm/rvc2V3YTvA7dExJ35/QuSLgYGAde2cO57wBaS/hQRbwGTmjowB+79gd0iYlbeNgz4ThvabGZmZjhwtqcplRcR0SBpGmkI90P7st7A5KptL5N60oqmVpX/CWAdoF9+/VpViPwXaYj41cI5zYqIhZLGkcLu4/n7tZUg24KrgSOAUcD/AK9GxP8BSPokqTdy19zmJaSe1jb1cJKueUdJexa2dQOmN3dSvr5dST29P5H0CnBBRFT3Blf0It3bKYVt1Z+VmZmZtYIDZ/vpW3mRh7wrwW9LUtgqmk4KYkX9+Wh46kMKopXy/w28QQqiC4C1IqK67KLqfU0dOxr4P0k/IQ3jf7GZMosqvaNfJk0LuLqw71hgO2AnYEoO4W/Q9N9cnUcKpgBIWhFYt7B/KvCriDiyxra9L08dmChpBdI82tskPRERL/PRe/IG8C7pfhfvvZmZmbWR53C2n0MkfTmvBj+etFDmniaOvQH4iqRBklaU9DXgMOCqquNGSOopaV1gODA2B8wAngEuyouHkNQrL45pzixg3cLcUgAi4lngOeA24MmI+GstFxwRc0lD72cDWwPXFHb3JAXkN0kLlU4D1mimuKeAnST1k/QJ0iKn4sr6S4F9JNVLWinfty3z4qsmSVpP0l6SVo+IxcDcvGtx/j6LNF+zck2LSXNhz8jn9gR+2lwdZmZm1jwHzvZzOXARafX0QNIcwLcbOzAiJpN6OIeSAtlY4NSIuLlw2GJSYP0z8DfgFVKvITl07kHqLXxK0jzScPgOLbTxYeBBYLKkuVVhbTTwJVpeLFTtamAX4P6ImFnY/jNSuPsHqadwIc0P8V9HWon/dD5+GjCjsjMi/kKaE3sMaVX/68AYWh6i7wYcCUzJ9+kS4KCIqLTlbOAASXMkTcjbjiYNo79Auv/j+SCgmpmZWSvVNTQ0+0QZq0H143vaobwdgIciYqlNech13gFsEBELl1a9XU3d+Yua/AfVcJxnsJiZWZfT1FS5D3EPpyFpFeA40qp2h00zMzNrV+5yWc7lVd/jSEPZP6naNwHYtrHzIqJ7+a2rjaSTgJOa2L1LRDT5GCQzMzMrn4fUzdqRh9TNzGw5U9OQuv8PaNaO7tp8AvX19R3dDDMzs07FczjNzMzMrFQOnGZmZmZWKgdOMzMzMyuVA6eZmZmZlcqB08zMzMxK5cBpZmZmZqVy4DQzMzOzUvnB72btqKkHv/uh72Zm1kX5b6mbmZmZWcdz4DQzMzOzUjlwmpmZmVmpHDjNzMzMrFReyWBLjaQTgR8CqwHfiog/LKV6LwMWRcTQpVGfmZmZfZhXqdtSIWkjYBrw2Yj4a0e3pyxepW5mZssZr1K3TqUvsOTjhE1JK7Vfc8zMzGxpcQ+nfYik7sBwYE+gFzAdOAx4ChiRt38SeBQ4KiKm5fMm5mP6AgOA14FjI+JOSQOBMcAqwALgtYjYRNLawKh8PMD9wA8j4q1c5hTgV8C3gK8CQ4AtgG2BAA4h/dL0E+A24Op83IvAARHxfC5nDGlIfUh+3wAcCRycy3sOGBwRL+T9PYBLgN2BecCpuR07R8TE5u6fezjNzGw54x5Oa5OrgK2AnYCewLeBmaRguHX+6gO8AYyXtELh3IOAC4DVgYuBayStGhE3AbsAiyOie0Rsko+/DlgT+H/5ax1gbFV7DgWOBXoAd+Zt2wF/B9YHDgBG5nYfCawFPA9c1MJ1Dgb2ynVOB35R2PdzoD8pjH4O2A1YATMzM2sTd7vY+yStC/wPaZ7l5Lz5JUndSGGyPiJm5GOPAd4Cvgb8Ph97U0Q8lvdfDvwM2Ax4ppG6NgD+E/h0RMzJ244FXpD0qYiYmQ+9IiL+mF+/IwngxYi4Mm+bIOlN4P5Cj+b1pDDbnJGF3tkxwLj8egVgf2CXiHg9bzsp3xczMzNrA/dwWlHf/P3Fqu29gE8AlRBKRMwnDZv3Lhw3s7B/QX7Zo4m6KudNLmx7uWofwJRGzp1Z9X5h1baFzdTbWBkLCsevA6wMTC3sL742MzOzVnLgtKIp+ftmVdtnA//mg0Bameu5Lmk4ui0q5/UtbOtftQ9gSRvLb6s3gHdJ0wYqNl7KbTAzM+tSPKRu74uI1yXdClwqaTCpZ68y3/Ja4CxJfwXmkuZqvgA82ca6/iHpAeACSQeRJh1fAEwoDKcvdRGxOA/JD5f0Z+BfpEVJZmZm1kbu4bRqhwB/Ah4hrdC+k7Q454ekleF/ID1P81PAtyNi8ceo64Bcx99I4XUuMOhjlNdejiZd44vAX4AHgQZSL6+ZmZm1kh+LZNYCSZuTAvGGEfGP5o71Y5HMzGw5U9Njkfx/QbMqkvqTenWfIC0iGgX8rqWwaWZmZo3zkLrZR60CXA68DfyZtOp9vw5tkZmZ2TLMPZxmVfKf3/xsR7fDzMysq3DgNGtHd20+gfr6+o5uhpmZWafiIXUzMzMzK5UDp5mZmZmVyoHTzMzMzErlwGlmZmZmpXLgNDMzM7NSOXCamZmZWakcOM3MzMysVA6cZmZmZlYqB04zMzMzK5UDp5mZmZmVyoHTzMzMzErlwGlmZmZmpXLgNDMzM7NSLVOBU9JGkhok9a3h2OGSHloKzWoTSZdJurij29GZdaZ7JOkhScM7uh1mZmbLohU7ugEdJYfWyUDviHi1huMHA6dExKbtUX9EHN4e5XQVkqaQ7u+4yralcY8kjQEWRcSQsusyMzNbXi1TPZy27JG0Uke3wczMzDpWp+7hlLQ+cDmwPfAacF7V/kOBo4HewCvAsIh4oHBInaRRwCDgHeDiiPhp3vdM/v43SQ3AuRFxVhPt2Aa4DFhZ0vy8efeImChp+9yuLYCZwKiIGF3DtY2h0LOW23AkcHAu6zlgcES8kPevBBwPHARsALyer/dWSSsCJwGDgTWBp4GjI+IvhbpWAN4D9gQWAMcBzwNX5PoC2D8i/pHPWRU4E9gLWB14EhgaES+1cF07AA/l6zgD6AX0kHQ0cASwITAHuI7Uo7lY0nhgY+BKSZcBj0XEgEbuUR/gIuAbpM/zNuDHEfFOC236EvAL4HPAYuAFYDfgUGD/fMw++fDVgSXAiaTPY1XgGqCuuTrMzMysaZ29h/M6UkDYGNiOFKiA98PmMFJgWBM4GbhdUnHIeztSUP0UsAdwrKT98r4v5O+bR0T3psImQET8HjgceCUf2z2HzX7AfcAvgbVz+0ZI2ruN1zuYFPDWAaaTQlLF2cABwN5AT1IIfzHvO54UqncF1gcmAQ9K6lk4/7ukgLYWcBYpaJ4JfAdYD2ggBcSKShDdOpf5BHB3jT2WK+S2fCmXDfAqsEtu+x7AIcAQgIioB6YBQ/K9HVBdYA7V9wCzgD65Xd8Azq+hPZcAD+RrXw84Fng3Is4j/YxdU/hcF5Pu8w9zO9cH3iD9LJmZmVkbdNoeTkkbAjsCm0bE28Dbks4gBQdIPZtnRkSlp/JeSQ8D+5DCGaQex3MjogF4StLlpFB3fTs1c1/g6YgYk98/Lmk0KUjd0obyRkbENHi/V3Jcfl1H6m0bGBHP5mNfzV+QehPPLfSGnpnbsBtwQz7mtxFxT95/LSkkj63MX5V0K6nHD0nrAPsBfSLitbztDOAYYCvg0RquZVj+3ACIiNsK+/4oaSywE9Bib3D2NWAzYKuIWAAskHQKcIekofkzbsq7pF9aekfEFODxFuoaBIyOiKcAJI0g/cJhZmZmbdBpAyewUf4+tbBtcuF1P+ASSRcVtq3IByEMYGpVEJlCGlJuL72r2gTwMqlnrC1mFl4vAHrk172A1figR7PZdkTEkrwIp3djZUfEQknV9S0s1Ncvf382H1exUlWZTVlC6qF9n6R9ST2L/Umf08q0HPyKegOzc9iseBlYhXR/Xm/m3IOBU4FHJb1HCvJnRMSiJo7fiPSzArx/P6c2cayZmZm1oDMHzhn5ex9SsADoW9g/FTg9IprrSewjqa4QOvvyQSBd0sr2NHb8dNLQcVF/qsJWO5hNCoSbAX9voh19K28kdcvv29qOSrjaLCJmt+H8hmLQl9SbFPL2BCZExLuSzgeKabalz2M60EvSqhGxMG/rD/yLdH+aFBGTSUP4SPocqZd8MvCrJuqdwYfvZx3p59DMzMzaoNMGzoh4VdJE4DxJBwOfBE4rHDIKGC7p76QFQKsAXwHeqAwtk+ZuHp8XDn2WNGR8bN43mxQ2NuPDvaJNmQWsK6lnRPwzb7sBOFXSINIw/ZeBw0iLY9pNRDRIupR0L6aRFhRtCKyVh9jHACdI+h2pZ24Y6bO9p431vS7peuBSScdExAxJawDfAh6MiPktFFGtO2m+8GzgPUlbAweSFi1VzCJ9Fk15EngJuEDSj4A1SHNRr25hOB1JB+V2/wOYCywizQ2u1Lu1pG4RUQmfY0n3+tfAn0kLrNav9WLNzMzswzr7oqH9gE+QercmAddWdkTEFaTV4VeTVj1PIw2bFhe1TCKFzlnA3cDPyfM388rmU4EbJM2VdHILbXkYeBCYnI/fPvec7QoMBd4kBZVTI+Lmj3PRTTgZuBm4A5gHTAQqC6RGksLvA6RFUjsCAwrBuC0OBf4GTJQ0jxS89iYtLmqViHgeOB24kxT4TuSDuaUVZwMHSJojaUIjZSwCdicNd08jBdAnSGGwJTuS5vAuAH5P+hkYm/ddSZqu8Gb+XFcg/Zz9AhhPup/rAr+r+YLNzMzsQ+oaGlqdH8ysCePHj2+or6/v6GaYmZktLTU9NrCz93CamZmZ2TKu087hXNokbQt8ZCg3OycizmlleSeRHsbemF0iYlJryutMCg+/rzYpInZZqo2h/T87MzMza18eUjdrRx5SNzOz5YyH1M3MzMys4zlwmpmZmVmpHDjNzMzMrFQOnGZmZmZWKgdOMzMzMyuVA6eZmZmZlcqB08zMzMxK5cBpZmZmZqVy4DQzMzOzUjlwmpmZmVmpHDjNzMzMrFQOnGZmZmZWKgdOMzMzMyuVA6eZmZmZlWrFjm6ALbskTQQeioizO7otFZJ+AuwHrA38C/gdcGxETCscswlwPrBj3vQ8sG1EvJf3C7gU+CwwEzg9IsYttYswMzPrYtzDaR1G0kolFDsW+GJE9AT6AtOAGwt19gImAc8AGwNrAUOBxXn/6sAE4DZgTeBw4DJJ25TQVjMzs+WCezitTSRdDGwLbCPpRGBGRGwu6VDgaKA38AowLCIeyOcMB7YDngYOBJ6WdAQwGRgMDAP6AI8A++f3hwBLgLMi4pKW2hURLxTe1uVzNy9sOxaYFhHDi6cVXu8JLATOi4gG4EFJvwb+F/h9S/WbmZnZR7mH09okIoaSegrPiojuhbA5jBQW1wROBm6XtGnh1O1Iw9S9gb0K2/cCvknqdewLPAG8DGwAHAxcKGnjWtomab//3969h1tVlXsc/75yURFRT0CZiBtSK49iyrDHTl6wvJVdyCQz0Ywy6H558tQhLSrLS5onO3pUNDAtTT1HCjTJTJ/Sp+yMvF9RZIMiBMolkBLRcf4YY8HYk7XWXmuz1trsvX+f59nPXnOOeRlzzDHnfNeYY67pnFsNrCUGv9Oy5COAZ51ztzjnVjjnHnLOnZyl7w/cn4LNkvvSeBEREekCtXBKI30J+K73/sE0fKtz7k7go0Cpn+dC7/2F6fP62F0SiIHrCgDn3BzgOO/99JT2G+fcSuAA4i3yqrz3vwB+4Zx7A/BJ4OEseShwEHAi8EFiADrbObfQe383sCOwurDIVcCQTrdeREREylLAKY00CrjEOXdxNq4/8Fw2vLDCvEuyz+sKw6VxO9aTGe/9UufcdOAZ59zIFNCuAf7kvb8pTXa7c+424APA3Sm9rbConYG/17NuERER2UQBp2yJ1wrDC4lPdN9YxzzN1h/YgXhrfgXwALBnmelKt9AfBMYX0g5I40VERKQLFHDKllhKx+DtImCac+4pYoC2HTAWeKHwME9TOOe2AT4L3OC9X+acGwH8BGgHSuu/HPijc2488GvgcOBo4LyUfjNwvnPuDODHxAejjgeOanb+RUREeis9NCRb4iLiz1aucs49mvpcng/MAFYS+1ueBTTj548qeS/wiHPuJeKDR+uAI733GwC8938m/k7necTb5z8BPu69/1NKX5WWMYHYl3M6MKWULiIiIvWzEELnU4lITWbPnh3e//73d3c2REREWsVqmUgtnCIiIiLSVOrDKT2Kc+4yYGKF5H3yV1iKiIjI1kEBp/Qo3vspxNdNioiISA+hW+oiIiIi0lQKOEVERESkqRRwioiIiEhTKeAUERERkaZSwCkiIiIiTaWAU0RERESaSgGniIiIiDSVAk4RERERaSoFnCIiIiLSVAo4RURERKSpFHCKiIiISFMp4BQRERGRplLAKSIiIiJNpYBTRERERJpKAaeIiIiINJUCThERERFpKgWcIiIiItJUFkLo7jyI9BrbbrvtI+vXr/9nd+ejt+vfv//QDRs2vNDd+egLVNatoXJuHZV1w70QQji2s4n6tyInIn3Ffvvt90/vvevufPR2zjmvcm4NlXVrqJxbR2XdPXRLXURERESaSgGniIiIiDSVAk6RxrqiuzPQR6icW0dl3Roq59ZRWXcDPTQkIiIiIk2lFk4RERERaSoFnCIiIiLSVPpZJJE6Oef2Bq4GXge8CJzqvX+qME0/4GLgWCAA53rvr2x1XnuyGsv5LOCjwKvAK8BU7/3cVue1p6ulrLNp3wzcD1zqvf9a63LZO9Ra1s65jwBnAUY8hxzpvf9bK/Pak9V4/hgOzAB2BwYAdwJf9N5vaHF2+wS1cIrU7zLgEu/93sAlwOVlpjkZ2BPYC3gHMM0519ayHPYOtZTzX4CDvPdjgEnAL51z27cwj71FLWVd+iJ1OTCrhXnrbTota+ecA6YBR3nv9wUOAVa3MpO9QC11eirweDp/jAHGAse3Lot9iwJOkTqkb8QHAtelUdcBBzrnhhUmPRGY7r1/zXu/nHiBntC6nPZstZaz936u935dGnyI2Br0upZltBeoo04DfAOYA8xrUfZ6lTrK+ivABd77pQDe+9Xee73BrEZ1lHMAdnTObQNsCwwEFrcso32MAk6R+uwOLPbevwqQ/j+fxudGAguz4UVlppHKai3n3KnAfO/9cy3IX29SU1k75/YHjgEuankOe49a6/U+wGjn3B+cc/c55850zlmL89qT1VrO3wP2BpYAS4G53vt7WpnRvkQBp4j0eM65w4kXj5O6Oy+9kXNuAPG3C6eULuLSVP2It3iPAg4H3gOc0q056p0mEO+M7ArsBhzmnDuhe7PUeyngFKnPs8BuqS9bqU/bG9P43CJgj2x4ZJlppLJayxnn3DuAa4Hx3vsnW5rL3qGWst4VeBNwq3OuHfgycLpzTj+gXZ96zh83ee9f9t6vAX4FvL2lOe3Zai3nLwA/T12fVhPL+YiW5rQPUcApUgfv/TLgATa1pJ0E3J/6aeZuJF6Qt0n9hsYDN7Uupz1breXsnDsI+CVwgvf+vtbmsneopay994u890O9923e+zbgP4l9lD/d8gz3YHWcP34BHO2cs9S6/G7gwdbltGero5wXEH9JBOfcQOBI4JFW5bOvUcApUr8pwBecc/OI35CnADjnbk1PlwJcAzwDPAX8Gfiu935Bd2S2B6ulnC8Ftgcud849kP72657s9mi1lLU0Ri1lfT2wDHiMGDg9ClzVDXntyWop5y8DhzrnHiaW8zxgendkti/Qqy1FREREpKnUwikiIiIiTaWAU0RERESaSgGniIiIiDSVAk4RERERaSoFnCIiIiLSVAo4RfowMzvGzP6YDY8zs/ZuzFLLmNlMM7uygctrM7OQDQ8zs4VmNrSGeaeY2TWNyktPYGaHmtmq7s5HX2RmE+s5zht9rEh1zTo2urDfzzWz7zVq/Qo4RfooMzPie7G/3cl0nzGzR8zs72a20sy8mZ2Ypbeb2cQy82023qJ5aVmDC2njzCyY2dr097yZzTCzf9myLe0eIYTlxB/w7qx8dwC+C0xrQba2GiGEP4YQdu7ufFRiZtPM7HfdnY++oFllbWZ3mdmZjV5usxWPjW6si+cBnzOz3RqxMAWcIn3X0cBA4M5KE5jZScSA6ZPATsTXw30FWNnFdR4BjAZeo/x7z18NIQwOIQwGDgHeQXyrTU/1U+ATZjakyjQTgYdDCPNblKcOzKyfmelaICIdhBBWAr8BJjdieTrJiLRAau0708zuTK13D5vZGDM7ycyeNrPVZnalmfXP5hlpZjeZ2VIzW2JmV5jZjln6D8zsmbS8+Wb25SytLbUWnmJmj5nZGjP7rZntmmVrPPC7UP3tD/8G/CGEcG+I/pG+ff+2i0UxGbiN+CamqiexEMIzwBzggGKamfVPZTK+MH6mmc1In99tZvemVtnlZna9mQ2vtL5UXodkw+PMbENhnVNTC+0qM7vHzKq+hSeE8BTwAvGVeZWMB24v5OVLZvZE2m+LzOwcM+uX0n5oZrMK049L0+6Qhvc1s7lpu0vzD0hppbrxSTN7DFgHDDezj5rZg6n1eYmZXV5aXprvDWY2O55Av0MAAAunSURBVNXVeWn+YGZt2TSnp9bw1WZ2v5kdXWmjy5TvTDO7xsx+msp3cTo+3mZm/5e2704ze2M2T7uZfcvM7k7HgTezg7L0qnXAzAakffpkWv58MzvBYgv+VGCcbWpxH11hOw5P61id9tnkLG2cmW0wsxPTsleb2Q35cVxmeV05V4wxs9+n7Xwmzd8vS397Kpu1ZnY38Utfvs5BZnaBmS0wsxVmdpuZ7Vkpj2Xy/Doz+5nFc9VSM7vasjsTVrjbkdXBEZXK2sxOS9v79VQfl5nZhWXq8YhsuaeZ2dPp838BhwJnpWU+WSHv08zsDjM7L9WRF83sq2a2RyrTNWb2VzN7azbPFh0rWV2fntX1zepN+ly1fArb0qHrQ4P2++3Ec9SWCyHoT3/6a/If0E58zeVbgQHAtcB84ApgB2Ak8VV2J6fptwOeJt5q3R7YBbgV+Gm2zInEFkcD3gX8AzgmpbUBgRiwDQWGAPcA07P57wW+WMjnOKA9G54A/BM4m/g+550rbNvEzsYDw4CXgeOJQWQAxhbWvSEb3hN4Mt/mwvLPB2Zlw4OBtcChafgQ4CCgP/AG4A/Addn0M4Ers+EAHFIlP99PZTYa6Eds9X0B2CUv8zL5nA2cXaVu/A34QGHch4FRad8ekKaZnNL2AdYDw7LprwauSp+HAy8SA/qBwG6AB75VqBt3pHIZmLbnPcC/Ehsi9iS+VvGcbB13AP+T6tJw4K60nLaUfjqxzu6flvHetD/2rLDdxfKdSazDx6X5p6T5fw2MAAYBv6djHW4HngfGpu34BrAcGFJjHTgvbeeYVNYjgDEpbRrxC1m143pUyvNpaR0HAyuACdk2BuJrKQcDryeeB77ZwHPFTql+nAVsm+Z7BjgjS38xlc3AVB5L6Xic/5x4rnh9muY7wBPAgHLHSpk830as57ukv1uAW6qcC9pSuYyoVNapTF8BLiGeA99EfPXk1HLLyOZ5Ohu+Czizk304La3nU2w6Dl4FflfYB7dn82zpsTKTWG8+kJZxfMrDHhWOjUrl83Rh3Mb91Ij9nqYZS7wjNbBaOdby19KLrv7011f/0gn3jGz4vekElAcNNwAXpc8nAPMLyxhLDNj6VVjHTcD56XPpZHxQlv454P5seB5wWmEZ4/ITUhr3PuB/iRe1V4m34PctbNtLwKrC32t0vMj8O/FCWbqI3QdcXlh3SPOuBBYAl1EmyE3Tv5UYeA1Pw5OAeVX2wfuAZdnwxpNzGq4YcBKDkTXAYYVlPlzaRioHnD8HLq2Sr/XAuE7qzwXADdnwvcBX0ucdU/m/Mw1/Dfh9Yf4Pky5OWd04rJN1fh74S/o8Is0zOkt/Nx0voo8ApxaWMZsKF3zKB5x5kDIoLX9CNu6zdKzD7cD3smEDFgEf66wOpGnXAsdVmHYanQecU4F7CuPOAeYW6nR+nP8QuLnKMtup71zxMeBZ0quq07jJwJPp88mpTPL075OOc+IX0gCMzNK3AVaTjgeqBJzEL70B2Csb9+Y0btdsm7oScL4MDMrGfYp0jBeXkc3TlYDz0cK4ZWX2wcoGHiszyep6Grcc+GCFY6NS+VQLOLd4v6dxe6Xphlcrx1r+NjbJi0jTLck+ryP2V1xeGFe61TYKGGmbP6kYiC01i83si8RWpRHEi+f2xIdUKq3zpWz5EIO6an0L4wpDmEP8FoyZvQW4FJhjZqNCOiMRW9+uzeez7GlIM7OU12tDCK+k0VcB55rZ10IIa9K4V0OND5KEEB43s/uILb0/Aj4BzMjWORb4AbHFbRCxjAaXWVQthqZ5Z1v2JDqx9WNE+Vk2GkIMnivZbD9Y7Dv7VWJran9i68Ofs0lmAJ8hPvT1EeC5EMI9KW0U8M5C3TFi602uvbDOo4BvAW8htpT1I154IbaSQryAlSwsLG8UcImZXZyN6w88R+021tcQwrpYbTY7boq3o9uzeYKZLSLtk07qwDBii+G8OvJXtDub79v5wAez4eJxXjwOy6nnXLE7sDA7Fkt52D19HlEmPc/zqPT/oVTeJQOyZVRTmiZf5vwsbQldtyyEsC4bbqfz460rinlcR5V614Bjpdw6a6kX9WjUfh/CpoaALaI+nCJbp4XEb/I7F/62CyEsNrN3Em8HTgaGpiBtNvGCWqv7ibdnaxZCeIIY5OxBvHVWq3cRbz1NKvXzIt6+GUxsoemqGcBpqd/RwcDPsrTria2oe4cQhlD+IaXcWmIAUvLG7PMLxAvCkYX9sUMI4dxOlrsvsawr6bAfzGx34i28s4ktRDsRbyvm+/Z6YG8zO5DY0jEjS1tIbA3J87lTiA9i5V7L1jkQmJWWOzKV19ezdS5O/0dm8+efS+udVFjv4BDCZ6pseyO0lT6kLzYj2RTkVqsDy4mBxF4VlvtahfG5Z/P1J6PT+FZ5FtjDOkYNeR4Wl0lvyz6XgqG9CvtuUAjhuhrXX1zm6ELaGiofW1C5rIeb2aBCvkv7tvQltSvL7bIGHSv1KrcdxTKFjtvfqP2+L7EFeH0X876RAk6RrdMcYKDFBxp2tGg3M/tQSh9CvL29HAhmdhyxX1E9ZhFv9VRkZpPMbIKl35JMHfSnAI+FEFbUsa7JxP5zbwHelv72JQZKn64z37nriYHsxcQ+VouztCHE20NrzGwksS9TNX8FPm5mA1Pn/q+WElIrwY+BC8xsLwAzG2zxd0yLF7mNUiA8jNgfrJJZdHyoaDDx3LwceMXMDgZOyWcIIawCbiYGpQcT+3CW/Axwad9tZ2bbpIcMjq2Sh4HElpqVIYR/mNk+xNuEpfU9R7w9eW6qj8OA4s/NXARMs/iQj5nZ9mZ2SGoVb6ZJZnagxYdJziC2ZN6S0irWgbRPLwXOt/iQlVl8iGVMmmQp8S7DwCrrvg4Ya2anWnyo7O3Eun5VQ7ewuluI+25qqrtvJgZApTzMIdapMyw+JHUgsf8xACGEZcQ7I5da+vkbM9vZzD5khZ8uKyeE8DzwW+DCNN8uwIXAb0IIpVa8vwInpWNmGLG/aa5SWW8DnJfq0mhid5Gr03pfJH3JsfhLC/sR76IUl1vzw081asSxUq9y5fMAMSB/XzrGPwQclqU3ar8fRTxHbTEFnCJboXQb6V3Elq8niBfNO4iBGsBcYmDxF2Lr2wnEAKQec4ENZjauyjQribduHzezl4h9B1cR+8LVxOJTweOBC0IIS/M/YivtAdbJ096VhBBWE7f7PcSfIMp9mtjnaw2xD+qNnSzu88SL0wpiH7mZhfRvA78CfmVmfyc+2DGF6ufRScDMlM9KrgH2TxdUQgiPZ+taRQySyrU0zSBu99zswk4q1yOIZd5O3Ic3U3hCNRdCWEvcz+eb2Vpii2qxe8bHiMHcc8QH0Erl+XJaxnTig1wz0joXEQOLAVW2vRGuIH7hWAmcSOyTWSrvzurAN4n7elaa5i42BSg3Elvollp8knhUYV5CCAuI/fs+T3xA4xrgrBDCDY3auM6kbT2a+KXlb2w6N/wopa8iPoh1IrGMLgb+u7CY04kP6N1lZmuIfZMnEG+l1mIisfyeJJ6vVgGnZulnEr8gLyGW8fWF+SuV9UJifVtAPPfcRqxjJR8nnotWp+0tBvoXEb98rTKzR2vclqoacax0wWblE+LPqH2JWP9XAMcSH1Qq5XOL97uZ7Uys35d1Md8dWMfb+yLSl6RWr6khhMPS8DhigNTWnfnqiVKr6IIQgqXhYcSnw12h/125eacQH/o5pdp0WxMzO4YYFG8fuulCYrGf8JnF/sPS85nZacR92+gWypbbGo6VrjCzc4j9hxvy4/l6aEikDwsh3EZsNZAGS0HmHjVOexkNakVoFjN7G7Ev2cPEBw7OBn7Zky6gIq3QW46VEMJ/NHJ5uqUuIrl2evabfbrTKuKDUL3VLsTb0muBu4GHiLf0RKQjHStl6Ja6iIiIiDSVWjhFREREpKkUcIqIiIhIUyngFBEREZGmUsApIiIiIk2lgFNEREREmur/Ac5tCGi0WUdpAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x396 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "shap.summary_plot(shap_values_mgbm_default, X, plot_type=\"bar\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Shapley for GBM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "First 5 rows of contributions:\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead>\n",
       "<tr><th style=\"text-align: right;\">  term_360</th><th style=\"text-align: right;\">  conforming</th><th style=\"text-align: right;\">  loan_amount_std</th><th style=\"text-align: right;\">  loan_to_value_ratio_std</th><th style=\"text-align: right;\">  no_intro_rate_period_std</th><th style=\"text-align: right;\">  intro_rate_period_std</th><th style=\"text-align: right;\">  property_value_std</th><th style=\"text-align: right;\">  income_std</th><th style=\"text-align: right;\">  debt_to_income_ratio_std</th><th style=\"text-align: right;\">  debt_to_income_ratio_missing.False</th><th style=\"text-align: right;\">  debt_to_income_ratio_missing.True</th><th style=\"text-align: right;\">  debt_to_income_ratio_missing.missing(NA)</th><th style=\"text-align: right;\">  BiasTerm</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td style=\"text-align: right;\"> 0.0119963</td><td style=\"text-align: right;\">  0.00246609</td><td style=\"text-align: right;\">        0.068955 </td><td style=\"text-align: right;\">                 1.05257 </td><td style=\"text-align: right;\">                -0.0164529</td><td style=\"text-align: right;\">             0.00692202</td><td style=\"text-align: right;\">            0.508051</td><td style=\"text-align: right;\"> -0.0513176 </td><td style=\"text-align: right;\">                 0.231107 </td><td style=\"text-align: right;\">                        -0.0001993  </td><td style=\"text-align: right;\">                                  0</td><td style=\"text-align: right;\">                                         0</td><td style=\"text-align: right;\">  -3.00977</td></tr>\n",
       "<tr><td style=\"text-align: right;\"> 0.0203195</td><td style=\"text-align: right;\">  0.00350931</td><td style=\"text-align: right;\">        0.0244885</td><td style=\"text-align: right;\">                 0.604385</td><td style=\"text-align: right;\">                -0.0219534</td><td style=\"text-align: right;\">             0.00749184</td><td style=\"text-align: right;\">           -0.360197</td><td style=\"text-align: right;\">  0.00489432</td><td style=\"text-align: right;\">                -0.369034 </td><td style=\"text-align: right;\">                        -0.00031517 </td><td style=\"text-align: right;\">                                  0</td><td style=\"text-align: right;\">                                         0</td><td style=\"text-align: right;\">  -3.00977</td></tr>\n",
       "<tr><td style=\"text-align: right;\"> 0.0118981</td><td style=\"text-align: right;\">  0.0041544 </td><td style=\"text-align: right;\">        0.127661 </td><td style=\"text-align: right;\">                 0.689605</td><td style=\"text-align: right;\">                -0.321384 </td><td style=\"text-align: right;\">            -0.415854  </td><td style=\"text-align: right;\">            0.602444</td><td style=\"text-align: right;\">  0.0187778 </td><td style=\"text-align: right;\">                -0.0260705</td><td style=\"text-align: right;\">                        -0.000213344</td><td style=\"text-align: right;\">                                  0</td><td style=\"text-align: right;\">                                         0</td><td style=\"text-align: right;\">  -3.00977</td></tr>\n",
       "<tr><td style=\"text-align: right;\"> 0.0136289</td><td style=\"text-align: right;\">  0.00478551</td><td style=\"text-align: right;\">       -0.0133384</td><td style=\"text-align: right;\">                -1.14775 </td><td style=\"text-align: right;\">                -0.118928 </td><td style=\"text-align: right;\">            -0.00935209</td><td style=\"text-align: right;\">           -0.507958</td><td style=\"text-align: right;\">  0.00600047</td><td style=\"text-align: right;\">                -0.191414 </td><td style=\"text-align: right;\">                        -0.000893107</td><td style=\"text-align: right;\">                                  0</td><td style=\"text-align: right;\">                                         0</td><td style=\"text-align: right;\">  -3.00977</td></tr>\n",
       "<tr><td style=\"text-align: right;\"> 0.0154476</td><td style=\"text-align: right;\">  0.00246609</td><td style=\"text-align: right;\">        0.102547 </td><td style=\"text-align: right;\">                 1.14454 </td><td style=\"text-align: right;\">                -0.0114345</td><td style=\"text-align: right;\">             0.0115538 </td><td style=\"text-align: right;\">            0.431013</td><td style=\"text-align: right;\"> -0.0103283 </td><td style=\"text-align: right;\">                 0.232346 </td><td style=\"text-align: right;\">                        -0.000154641</td><td style=\"text-align: right;\">                                  0</td><td style=\"text-align: right;\">                                         0</td><td style=\"text-align: right;\">  -3.00977</td></tr>\n",
       "</tbody>\n",
       "</table>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "shap_values_gbm_default, expected_value_gbm_default = get_shapley(gbm_default, train, X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAApwAAAFeCAYAAAAog8+HAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xm8XdP9//HXjaFKEmNQEkkM5aeDat+tamsovvl+Dbda6htTIzS+KEV9adTUGCqN0KjSiqFCYh5agphaUb6mflBapYpMNCFIUkm0keT+/ljrsB13OPe4Oze5eT8fj/u45+y191pr73PD+6619r4NTU1NmJmZmZmVpVtnd8DMzMzMujYHTjMzMzMrlQOnmZmZmZXKgdPMzMzMSuXAaWZmZmalcuA0MzMzs1I5cJqZmZlZqRw4zczMzKxUDpxmZmZmVqoVO7sDZl3J+PHjmxobGzu7G2ZmZktKQy07eYTTzMzMzErlwGlmZmZmpXLgNDMzM7NSOXCamZmZWakcOM3MzMysVA6cZmZmZlYqB04zMzMzK5UDp5mZmZmVyoHTzMzMzErlwGlmZmZmpXLgNDMzM7NSOXCamZmZWakcOM3MzMysVA6cZmZmZlYqB04zMzMzK5UDp5mZmZmVqqGpqamz+2DWZTScu9D/oMzMbKnUdPyKZVTbUMtOHuE0MzMzs1I5cJqZmZlZqRw4zczMzKxUDpxmZmZmVioHTjMzMzMrlQOnmZmZmZXKgbMTSZoo6ZTO7oe1TdKLkgZ3dj/MzMyWRQ6ctsyRtKOkhR1Ul0O/mZlZyRw4zczMzKxUpTxy3tpP0meB84GtgVnAr4HhEbEol18B7AKsAUwDzoqIa3LZjsB9wAHA2cA6wN3AdyPi7Tba3Qq4APgUsALwKHBURLyUy8fk7e8CewHzgOOB54BLgS2AAA6IiH/kY9YGRgEDcjN3Az+IiLdy+WTglIgYl9/3AyYBfSLilUKb/wL2yW2eERGjJW0ATABWkDQ3139kRFzZyjnuAowENgEWAH+KiF0kXQhsB2wr6UTg1YjYXNJKwAjgQGBxPhczMzOrk0c4lwKSVgfuBe4H1gd2Bw4Bjivs9hDwOVLgPAMYI2nLQvkKpIC3FfBJUnA9uobmm4BhwIZAP2AuMK5qn28DNwNrAWeSguYZwLeA9XIdpxf2vxpYE/h/+WsdYGwNfaluc3xu8/vAhZL65lC7K7AoIrrnrxbDZnYVKVSvns/zLICIOAp4EDgz17N53v9EYA/gK0B/0nXp287+m5mZWeYRzqXD7qSRt7Miogl4TtIIUuAcCRARlxf2v07S8cCOwF8L20+MiLnAXEm/BdRWwxHxTOHtvyWdDvxZ0qoRMT9v/31E3AEg6SrgV8DYiHglb7sJODS/3gD4T+CTETErbzsOeF7SJyJieo3X5PcRcVt+fYuk2aTAPaXG44sWkEY314uIGcDENvYfBPw0Il4EyNf6u3W0a2ZmZjhwLi36AFNy2Kx4KW9HUjfSKORA0ghoE7Aa0Kuw/6KImFl4Pw/o0VbDkjYhhdpt8v6VPvTi/XD3XkiMiPmSPrANmF9oq0/+PqnqXCpltQbO6v1qOp8W7AmcRArSM4FLIuL8VvbvDUyuvImIeZJer7NtMzOz5Z6n1JcO04C+khoK2zbO2wH2A4YAewNrRsQawNNAAx/dxcDbwGcjoifw1by93rorfe5X2LZxVdnbpMBcsUE721jcnp0j4umIGAisCxwGDJe0Uyt1vUqh/5Kqw72ZmZm1gwPn0uEO4GPASZJWlrQ5MBSoTKP3BBYCM4Fukg4hrdXsCD1Jo4ezJa1DWptZt7zG8h7gPElrSFoTOA+YUJhOfwLYT1J3Sb2AU9vZzAzSTUP929oxX8+DJK2TR5BnkULmokJdm1YdNhY4QdImkj4OnIP/rZiZmdXN/xNdCkTEHNINP7sAr5Hu6r4K+Fne5UrgMeBF0ujblqSbXTrCD0h3av8z13l7B9R5IGkU82/A88Bs0rrIilNIgW86aT3lde2pPCJeIK0jfVzSbEnfaeOQgaQ1pHOB24AfR8QDuWwUoFzPs3nbcNJn8ChpacBU6ls7amZmZkBDU1NT23uZWU0azl3of1BmZrZUajq+lFt3alqC5xFOMzMzMyuV71Lv4iRtR3pQenPOjoizl2R/yiDpAGB0C8WHRcTVS7I/ZmZm9kGeUjfrQOPHj29qbGzs7G6YmZktKZ5SNzMzM7PO58BpZmZmZqVy4DQzMzOzUjlwmpmZmVmpHDjNzMzMrFQOnGZmZmZWKgdOMzMzMyuVn8Np1oH8py3NSvvzeWa2dPJzOM3MzMys8zlwmpmZmVmpHDjNzMzMrFQOnGZmZmZWKgfOAkkTJP2ws/uxPPio11rSgZImd2CXWmvrMkljlkRbZmZmXZFvJSyIiF1r3VfSROC+iDirvB61n6TBwCkRsWln96U17bnWZZE0DPhaROzS2X0xMzPryjzCWSJJKy3N9XWGrnAOZmZm1j4e4SyojFoC44BJwCDgR0Af4BHgoIiYLulCYDtgW0knAq9GxOZ52nUl4F3gG8D1wBGSdgDOAbYApgOjImJ0jf35E9AP2Ak4W9I44DLgC8DKwDPAsRHxhKRtgYuBlSXNzdXsERETJX0aOA/4PPAOcDVwWkS820YfKue0GNgTmAmcGRFjCvtsBwwHtgRmAb8EfhYRTZJ2zNf0YOB0oBfQo3qEWNJngfOBrXMdvwaGR8SiXP6lXO8W+Zrc09b1y8f1A0YD2wBNpM91P+BzwElAt8K1+mxEvCzpEODk3NdbSc8YW1hLe2ZmZvZhHuFs3UBge2BDYDXgDICIOAp4kBS8ukfE5oVj9gEmkMLK/0rqD9wF/ApYGxgMDJe0T419OAS4AFg9f+9GCl59gfWBJ4FbJK0UEY8AhwMv5351z2FzXeAB4JZ8LtsC/0EK07X4b+BuYC3gMOBXkr4CIGlL4E5gZD7n3YGjgO8Ujl8B2I0UJterrlzS6sC9wP35nHbP531coXwCcFPuww+A79XY97OBqbnddUjXf1ZEXJ/LJhau1cs5PF9Euo5r5X4NrLEtMzMza4ZHOFt3ekS8ASDpGmBIDcc8lMMMwHxJ+wFPFkYEH5U0Otd1Yw313RQRv6/URwpPUyuFkk4BjgY2A/7aQh2DgKcLo6qvShoOjCCH6DY8GhHj8uv7JN1MCm4Pk4LfjRFxay5/Po8ADwKuKtQxNCLmtFD/7sAC4KyIaAKekzSCFDhHAnsA84ARufyPki4HDqih7wtIIXbjiHiONCLcmkGka35vfn+VpMNqaMfMzMxa4MDZuumF1/OAHjUcM7nqfR/SNG7RS6Tp6Vp8oD5J6wA/A3YE1iBNdUMaXWxJf+CrkmYXtjWQRh7b3Yf8/vOFuneStFehvBswrfB+cdX7an2AKTlMVryUtwP0bqa8+pq25ATgVGC8pNVIo6Q/ioi5LezfG4iqbbW2ZWZmZs1w4Kzf4hq3TyNNJxdtTOsBrLX6hgOfALbJ60l7AP/k/b9l2ly/ppDWS+5eY5vV+jXz/pVC3b+OiCNbOb6pKixWmwb0ldRQ2K94jV5tpry6T82KiJmkEeCjJW1MWpP5Q+A0mr9WrzZTdz/gxVraMzMzsw9z4KzfDKCWRw9dC5wqaRBwDWlk8DDgiDrb7UmaWp8lqTtpWry6X+tK6hkR/8zbriKtJz0k92EBKUR9MiLuqqHNL+elATcAOwB7A5VHCf0SeEDSXaS1qk3AJ4FeEfFAjed0B+mGoZMkjSSNmg4l3ewDcDtp/eoJkkYBnwG+C/y7rYolDQQeJ43KziGd+6JcPAPYSNLKEbEgbxsL3JVvlnoA2Jd0w5EDp5mZWZ1801D9RgGSNFvSsy3tFBGTSCOcRwFvkgLNqRFxQ53tngasm+t6hrSOclGh/H7SjS6Tct92iIgZwNeBb5KC1yzgN6RRxFrckM9hFnA5cGRE/F8+v7+Q1lgeS1qC8Dowhtan+D8gr+0cQAqxr5FuULqKtHSAiJhNWuc5MPfhAtJNWLXYmhQc5wLPkm6yGpnLbiSNos7I16p/DsnfJz0J4C3gv0hPGzAzM7M6NTQ1tTbTacu7PNK3MCJquWFquddw7kL/g7LlXtPxnjwzW440tL2LRzjNzMzMrGT+NbQTSTqJ9PDx5uwaEQ8ugT4cwPtrJast9Y8DkrQRLT8OalxEHL4k+2NmZmYf5il1sw7kKXUzT6mbLWdqmlL3fxXMOtBtm0+gsbGxs7thZma2VPEaTjMzMzMrlQOnmZmZmZXKgdPMzMzMSuXAaWZmZmalcuA0MzMzs1I5cJqZmZlZqRw4zczMzKxUfvC7WQfyg9+7Hj/E3MysVf5b6mZmZmbW+Rw4zczMzKxUDpxmZmZmVioHTjMzMzMrlQOnmZmZmZXKgdPeI2mCpB92dj+WNpIukzSms/thZma2rPLzPuw9EbFrZ/dhSZI0DPhaROzS2X0xMzPryjzCaWZmZmal8ginvUfSROA+YBwwCRgE/AjoAzwCHBQR0/O+3YFhwF5AL2AacFhEPChpVWB4Lvs48BBwdERMLbTzJNAf2AV4Hfgf0sNjzwc2An4HDIqIt/MxawPnAAOAVYD7ge9HxGttnFM/YDSwDdCUz2s/4HPASUA3SXPz7p+NiJclHQKcnM/r1tyvhe24lGZmZlbgEU5rzUBge2BDYDXgjELZ5aQQtzPQE/gGMD2XjQK+nL/6Am8A4yWtUDj+O8BPgTWA64GxpNC5PdAP2Bw4GkBSA/BbUmD8dK7zbeCaGs7hbGAqsB6wDjAYmBUR1+eyiRHRPX+9LGk74CLgcGAt4N58HczMzKxOHuG01pweEW8ASLoGGJJfrwv8N/DpiJiU930xl3UDDgIaI+LVvO1Y4C3gS6SRUoAbIuKxXD6ONJI6MiLeyttuB5T3/UL+2iUi/p3Lfwi8Ial3RLzSyjksANYHNo6I54Bn2jjnQcBNEXFvfn+VpMPaOMbMzMxa4RFOa830wut5QI/8ul/+/kIzx/QCPkaaugYgIuaSps37tFD3/Ba2Vdrrn+t8TdJsSbOBl4B/kabfW3NC7st4SdMl/SIvB2hJb2By1bZJzexnZmZmNfIIp9Vjcv6+GfDXqrKZwL9JobQy6tkdWJe0zrMeU0iBd62IWNyeAyNiJmlq/mhJG5PWZP4QOA1orq5XeT9QV/Qjn4uZmZm1nwOntVtEvC7pJuCXkgaTAuEmuexFSVcBZ0r6KzAbOA94Hni83iaBp4ELJP04It6U1AvYOSKua+1ASQNzu5OBOaQp9kW5eAawkaSVI2JB3jYWuCs/d/MBYF/SWlUHTjMzszp5St3qdQjwJ1Ioe5s0crh+LvsBKST+kXTDzieAb0TEombqaVMe1dyTdLf4E5LeBh4Fdqzh8K1zH+cCz5Lujh+Zy24kjbrOyFP1/SPiAeD7wGWkdaf/RbqpyczMzOrU0NTU1Nl9MOsyGs5d6H9QXUzT8Z4IMjNrRUMtO3mE08zMzMxK5V/dbZkmaSM+fONSxbiIOHxJ9sfMzMw+zFPqZh1o/PjxTY2NjZ3dDTMzsyXFU+pmZmZm1vkcOM3MzMysVA6cZmZmZlYqB04zMzMzK5UDp5mZmZmVyoHTzMzMzErlwGlmZmZmpfJzOM060NL4py39pxnNzKxEfg6nmZmZmXU+B04zMzMzK5UDp5mZmZmVyoHTzMzMzEpVV+CU1FtSk6R+New7TNJ99bSzJEi6WNKFnd2PpVlnXqOOaFvSAZKe7qg+mZmZWft0+u2rObROAvpExCs17D8YOCUiNu2I9iPi8I6op6uQNJl0fcdVtnXmNeqItiPiauDqDuiOmZmZ1cFT6ssJSSt1dh/MzMxs+VTTCKek9YFLgB2A14BzqsoPBY4B+gAvA0Mj4p7CLg2SRgGDgHeACyPip7msMtX5N0lNwIiIOLOFfmwLXAysLGlu3rxHREyUtEPu1xbAdGBURIyu4dzGAAsjYkh+3wQcCRyc63oWGBwRz+fylYATgIOADYDX8/neJGlF4CRgMLAm8CRwTET8pdDWCsC7wF7APOB44Dng0txeAAdExD/yMasCZwB7A6sDjwNHRcSLbZzXjsB9+TxOB3oBPSQdAxwBbAjMIo38nRIRiySNBzYCLpN0MfBwRAxo5hr1BS4Avkr6PG8GfhQR77TRp2HAdvkcDyH9wvOTfPwVwBeBF4ADI+K5wjVbGBFDJDUAZ+Vz6gG8CZwXEb+QtCbpZ3Qn0s/1K8DhEfFg9ai4pInAE0A/YADpMzwuIm7N5Q3Aj4DvAasCVwKfBR6MiGGtnaOZmZl9WK0jnFcDi0hhZHtSoALeC5tDgQNIIetk4BZJxSnv7UlB9RPAnsBxkvbPZVvl75tHRPeWwiZARDwCHA68nPftnsNmf+Au4FfA2rl/wyXtU+P5VRtMCnjrANOAXxTKzgIOBPYBepJC+Au57ARSqN4NWB94ELhXUs/C8d8mBay1gDNJQfMM4FvAekATKSBWVILol3OdjwG31zhiuULuy9a5bkhBbNfc9z1JwW8IQEQ0AlOBIfnaDqiuMIfqO4AZQN/cr68C59bQH0g/C3/P53IgMBK4nBTy1yKF7wtaOPY/SEF/m4joAXwJeCiXnUAKh32BNUjXs7UlGgcB55FC/IXAlTncA3yH9AtUI+m6Tc/9NjMzszq0OcIpaUPSqNGmETEHmCPpdKAygnkMcEZEVEYq75R0P7AvKZxB+h/2iIhoAp6QdAkp1F3TQeexH/BkRIzJ7x+VNJoUpG6so76RETEV3hthG5dfN5CC0cCIeCbv+wrvB5uDSedZGQ09I/dhd+DavM/vI+KOXH4VKSSPraxflXQTcGh+vQ6wP9A3Il7L204HjgW24f2w1Zqh+XMDICJuLpQ9JWkssDPQ5mhw9iVgM1LomwfMk3QK8FtJR+XPuDUvRMRl+fUESW8CdxdGNK+h5fWWC4BVgE9JmhkRr5NGJytlawObA09FxAst1FFxfUQ8nNu8BPhZPq+nSb80jI6Ip3L5SNLnbmZmZnWoZYSzd/4+pbBtUuF1f+AiSbMrX8DXSVO2FVOqgsjkQr0doU9VnwBeytvrMb3weh5p+hbStPRqvD+i2Wo/ImIx6VyL/ZheKJ/fTHvzC+31z9+fKVzbt4CVqO3cFpNGaN8jaT9Jf5T0pqQ5pCDVq4a6KvoAM3PYrHiJFARrqWd61fv5tHz+HxARE0lLFk4BXpd0jyTl4pHA70jT3zMlXSlpvebqqe5H4Vwq7W5I4ec9/+x+4DqamZlZ7WpZw/lq/t6XFCwgrX2rmAL8OCJaG0nsK6mhEDr78f6o4OLauvqe5vafRpo6LtqYjg8JM0mBaDPStHBz/ehXeSOpW35fbz8qoWeziJhZx/FNxaAvqQ9ptHYvYEJELJB0LqDCMW19HtOAXpJWLQTmjYF/ka5PqSLiEuCSPP09DLgF2CiHxpOBk/Oa43GkEDqojmZeJf28A++NbNf7y4uZmdlyr83AGRGv5JsszpF0MPBx4LTCLqOAYZL+TpqOXAX4AvBGZWqZtHbzhHzj0KdJU8bH5bKZpJCzGa2vuauYAawrqWdE/DNvuxY4VdIg0jT954HDSDfHdJiIaJL0S9K1mEq6oWhDYK08xT4G+KGkP5BGNoeSrvEddbb3ep5i/qWkYyPiVUlrkEaQ742IuW1UUa07aVR7JvCupC+T1is+V9hnBumzaMnjwIvAeZL+l7Re8kzgihqm0z8SSV8CPpb78G/gbdLaYiQ15n69AMwlBeBFdTY1Fhgh6RbSZ3w06QYxMzMzq0OtNw3tT/of/TTSjTBXVQoi4lLS3eFXkO56ngqcSpr2rXiQFDpnALcDPyev38x3Np8KXJunjU9uoy/3A/cCk/L+O0TEJNII51GkO5fHAqdGxA01nl97nAzcAPyWFHgmApUbpEaSwu89pJukdgIGFIJxPQ4F/gZMlPQ28GfSDUvtDnd5neSPgVuB2cCJvL+2tOIs4EBJsyRNaKaOhcAepCURU0nh7zHS3fZl60762XmD9DkPAAbmsk2A8cA/SWH/HVLgr8dVwEXAnaTPsTfwKCnkmpmZWTs1NDWVOihltszLSyOmASdERKs3ujWcu3Cp+wfVdHyn/30HMzPruhpq2cn/JzJrhqR9SaPY3UjP5FwV+NCIr5mZmbVtqQuckraj5f+xnx0RZ7ezvpNIdzY3Z9eIeLA99S1NCg+/r/ZgROy6RDtDx392newo3n9U1F+A3SJiVif2x8zMbJnlKXWzDuQpdTMzW854St1sSbtt8wk0NjZ2djfMzMyWKrXepW5mZmZmVhcHTjMzMzMrlQOnmZmZmZXKgdPMzMzMSuXAaWZmZmalcuA0MzMzs1I5cJqZmZlZqfzgd7MOtCQe/O4HuZuZ2VKkpge/e4TTzMzMzErlwGlmZmZmpXLgNDMzM7NSOXCamZmZWakcOM3MzMysVA6cZmZmZlaqLhk4JU2UdEpn92NpIqlJ0tc6ux/tIelZSQM7ux8AkhZK2rGz+2FmZrYs8gP9SiJpMHBKRGza2X1Z2knqB0wC+kTEK5XtEfGpJdD2ZNLnNK7stszMzJZXXXKE05Yeklbq7D6YmZlZ5+ryI5ySPgucD2wNzAJ+DQyPiEW5/ApgF2ANYBpwVkRck8t2BO4DDgDOBtYB7ga+GxFvt9LmtsDFwMqS5ubNe0TEREk7AOcAWwDTgVERMbqNc9gduALYMCLezdu6AzOA3SPiAUlnA/sC6wKvAb+IiPNbqG8wVaOvksYACyNiSH6/EfAz4GtAEzAe+N/WzjsfNwzYHngS+E7+vmtr1xl4On//m6QmYEREnFk9+ljPtcvH7Qv8GOgNzAfuioiDJI0HNgIuk3Qx8HBEDJDUA7gQaATeBk5rqw0zMzNrWZce4ZS0OnAvcD+wPrA7cAhwXGG3h4DPkYLQGcAYSVsWylcABgBbAZ8kBdejW2s3Ih4BDgdejoju+WuipP7AXcCvgLWBwcBwSfu0cSp3AQtz/yv2IQXOP+T3fyWFwx7Aobne/2yj3mZJWgX4fa6zP7AlKaz9vMYqticFwj7A3nlba9d5q/x983ytzmymT3VdO0mrAmOBIyOiB7AxcBlARDQCU4Ehud0B+bDzgc3yeX8W2JP0c2BmZmZ16OojnLsDC0ijaU3Ac5JGkALnSICIuLyw/3WSjgd2JIWtihMjYi4wV9JvAdXZn/2AJyNiTH7/qKTRwBDgxpYOiohFksYCBwO/zZsPBq7I50XVGsTfS7oD2Jk0ItteewANEVEZ2XtH0qnAw5IOrYwOt2JKRJyXXy/I/avlOremrmuXvQtsIelPEfEW8GBLO0rqRhrR3j0iZuRtQ4Fv1dhPMzMzq9LVA2cfUvhpKmx7KW+vhIthwEDSCGgTsBrQq7D/ooiYWXg/jzSKWG9/JlVte4k0gtaWK4BnJK2b2/8KsH+lUNLRpJHN3kAD8HHgmmbqqUV/YCNJs6u2N5Gu06ttHD+l+KbG69yWuq5dRMyXtBvpl4yfSHoZOK8wnV+tF/AxYHJhW3W7ZmZm1g5dekqdtFawr6SGwraN83ZIo2ZDSNO+a0bEGqT1hA18dItb6E+/qm3F/rQoIp4HngAOJE0n31e5o1vSV4ERwGHAOvk8xtPyebxNCnxFGxReTwFeiIg1qr5WiYi2wiZ8+Nzbus7NXatqH+XaTYyIb5DW4J4FjJO0SQttv0EalS22Vd2umZmZtUNXH+G8g7Qe7yRJI0kjd0OByo0mPUlrI2cC3fLNNFsBt3dA2zOAdSX1jIh/5m3XAqdKGkQaffw8KSQeUWOdV5DWj/YATihs7wksIp1HU77JaFdanmr+U+7bHsCdpFHC7YHKtPztpNHAk4BfAHNJgfRLEfGbGvta1NZ1nkkKfpsBrzRXAXVeO0nrkda23hcRcwqjtpVlATNyu8B7yxeuAU6X9BfgHeCntZ+qmZmZVevSI5wRMYd0w88upDu37wauIt19DXAl8BjwImmaeEtaWd/XTveTbliaJGm2pB0iYhKwG3AU8CbpZpZTI+KGGuu8jjSq1x24tbC9cl6Pk0bovg20GAwj4iXgGOAS4C3gv4CbC+XzgZ1I1+N5YA7wO9JNP/Vo9TpHxDvAqcC1+Vqd3Eyf67123YAjgcmS3gYuAg6KiMm5/CzgQEmzJE3I244hTaM/D/yZNFrc1rpVMzMza0FDU1NT23uZWU0azl1Y+j+opuO7+sSEmZktQ2pahtilRzjNzMzMrPN5qKROkrYDJrRQfHZEnN3O+k4CTmqheNeI6Kip/o+ko8+7IyxN1+62zSfQ2Ni4pJozMzNbJnhK3awDjR8/vsmB08zMliOeUjczMzOzzufAaWZmZmalcuA0MzMzs1I5cJqZmZlZqRw4zczMzKxUDpxmZmZmVio/FsmsA3XUXxryXxMyM7NlhB+LZGZmZmadz4HTzMzMzErlwGlmZmZmpXLgNDMzM7NSOXCamZmZWakcODNJcyVt29n9WF581Ost6TJJYzqwS6219aKkwUuiLTMzs67Iz17JIqJ7e/aX1ARsFxEPldSluuQQtjAihnR2X1rT3utdBkkTgfsi4qzO7ouZmVlX5hHOEklaqQPrapC0zP+C0JHXxMzMzJYNy1yAkTQZuATYGdgGmAz8T0Q8nAPZScBgYE3gSeCYiPhLDfW+N2KZp09PAS4AfgisBtwAfC8iFkl6Oh92j6TFwHURMST37dfA14EvAkOA6yQdARwLrA88B5wQEQ/W2Kdjge8AnwK+Lmk14Gzgk8BC4HfA0RHxuqQfAgfkY/fN1aye+/xN4FRgE2A6cFZEXF1DHyrnNAD4HPA8cERE/LGwz6HAMUAf4GVgaETck8uGAduTPovv5O+7Vo8QS9obOA3oR/pMh0XEbwptHAKcDPQCbiU9aHZhDf3fBRiZz3sB8KeI2EXShcB2wLaSTgRejYjNcyAeARwILAZGtdWGmZmZtW5ZHeE8BDgaWB24F7gybz8BGATsRgp3DwL3SupZRxt9gfVIQeWLwD7AvgARsVXeZ0BEdK+avj4UOA7oAdwqaT/gzNyvtYFLgbsk9a2xH98FBgLdgaeAfwNHkYLXZ4ANgJ/nfp0DXA1cmfvVPYfN/wAuJ4XXtYBm9CKHAAAgAElEQVSDgAslbV9jHw4nBcq1gJuAOyvXNIfNoaSguyYpFN4iadPC8duTQm4fYO/qyiV9Jff7RNI1Ogm4VtI2uXw74KLcj7VIn/nAGvt+FekXh9WBDYGzACLiKNLPx5n5Om2e9z8R2AP4CtCfFIBr/azMzMysGcvcCGc2OiKehXTzCHCspNWBg4EREfF8LjuDNMq4O3BtO9t4BzgtIhYBL0r6HSBSMGrNpRHxVKUOSQfn/j6Wt10uaQiwPzC8hn6cGxEv5deLgOKa0RmSziGNQLbmGODnhVHVxyWNI4XgP9TQh8sj4gkASSOA75FC2TW57jMiojLqe6ek+0nhvLI2ckpEnJdfL2im/sHAzRExIb+/Q9JvSL9YPJb7eVNE3JvLr5J0WA39rrS3CbBeRMwAJrax/yDgpxHxYj7f40mh38zMzOq0rAbO6YXX8/L3HqQRtEmVgohYnKeE+9TRxus5bBbb6VHDcZOr3vchTccXvdSOPn2gPklfIE2pbwWsSppabusGnP6k6fjjCttWII3wtasPEdEkaSrQu1D3RZIuKOy/IvBK4f2UNurvAzxRte0l4PP5dW8gqsonUZs9SSOmf5Y0E7gkIs5vZf/efPB850l6vca2zMzMrBnLauBsyTTSFCgAkrrl99NKaKuphe2LW+tTtjEwvsZ2quu7jjStvU9E/FPSHlV1Ve8PKfCNiYiRNbZZrV/lhaQGYCPeD5RTgB9HxI2tHN9cn4paukaVz+3VZsr7AS+2US955HVg7vfXSOtun4mI37fQrw+0ldfM9mqrHTMzM2tZVwucY4AfSvoDaZRqKOkc7yihrRnAZnxwirulPv1c0m28f+PM54D96my3JzAHeFvSRqQ1h9X9+rKkbhFRCVTnA2MkPQo8TBrd/AzQEBHVI4fNOSRPcf8Z+AFpZLVyTUcBwyT9HXgaWAX4AvBGZWlDDa4E7pM0FriPdIPSXsCOuXwsad3rGOAB0nT9NrQROCWtTLrOd0TEG5JmkUJmZeR6BrBp1WFjgRPyI5P+AZzDsrvW2czMbKnQ1f5HOpK0VvMe4DVgJ9KNPf8soa2TgTMkzZI0uqWdIuIa4HRgHPAmcASwW0S0Nc3ckv8hrUt9G7gFqB5ZvIx0V/2bkmZLWiHfMX4o6fq8QVqSMIq2p+IrLiHdeDOLdLPO7hExJ5/fpaRQdkUun0q6G77mxx9FxP+RbmQ6N9dxDnBgRDyayx8Avp/P7S3gv4Dra6x+IPC8pLnAbaTR2Ady2ShA+To9m7cNB+4GHiVN20+l7SUBZmZm1oqGpqaWZobN3nss0ikRMa6z+7IsaDh3YYf8g2o6vqtNPpiZWRfVUMtOXW2E08zMzMyWMsvNMEqeMm3ueYpTIuJTS7o/AJImkB4+/iFL6k8/SrqY9JDz5my5JPrwUUg6AGhpScNhtTzc3szMzMrlKXWzDjR+/PimxsbGzu6GmZnZkuIpdTMzMzPrfA6cZmZmZlYqB04zMzMzK5UDp5mZmZmVyoHTzMzMzErlwGlmZmZmpXLgNDMzM7NSOXCamZmZWan84HezDlTv31L33043M7NllB/8bmZmZmadz4HTzMzMzErlwGlmZmZmpXLgNDMzM7NSOXCamZmZWakcOM3MzMysVH4Wi9VN0kTgvog4q7P7UiHpJ8D+wNrAv4A/AMdFxNTCPpsA5wI75U3PAdtFxLu5XMAvgU8D04EfR8S4JXYSZmZmXYxHOK3TSFqphGrHAp+LiJ5AP2AqcF2hzV7Ag8DTwEbAWsBRwKJcvjowAbgZWBM4HLhY0rYl9NXMzGy54BFOq4ukC4HtgG0lnQi8GhGbSzoUOAboA7wMDI2Ie/Ixw4DtgSeB7wBPSjoCmAQMBoYCfYEHgAPy+0OAxcCZEXFRW/2KiOcLbxvysZsXth0HTI2IYcXDCq/3AuYD50REE3CvpN8A/wM80lb7ZmZm9mEe4bS6RMRRpJHCMyOieyFsDiWFxTWBk4FbJG1aOHR70jR1H2Dvwva9ga+RRh37AY8BLwEbAAcD50vaqJa+Sdpf0hxgLin8DisUfx2YJukOSW9JekbSAYXyrYCnctiseDJvNzMzszp4hNM60jHAGRHxdH5/p6T7gX2ByjrPKRFxXn69IC2XBFJwfQtA0u3A7hFxaS6bIGkWsDVpirxVEXENcI2k9YHvAn8uFK8DfBEYCOxJCqDjJU2JiIeAHsCcqipnAz3bPHszMzNrlgOndaT+wEWSLihsWxF4pfB+SgvHTi+8nl/1vrKtR3s6ExEzJF0KvCxpoxxo3wYeiYib8m73SroL+AbwUC7vV1XVGsA/29O2mZmZvc+B0z6KxVXvp5Du6L6xHceUbUVgNdLU/FvAn4BNm9mvMoX+NPDNqrKt83YzMzOrgwOnfRQz+GB4GwUMk/R3UkBbBfgC8EbVzTylkNQN+B5wQ0S8Lqk38AtgMlBpfzTwoKRvArcBOwADgBG5/DfAOZJOAH5OujFqL+A/yu6/mZlZV+WbhuyjGEV6bOVsSc/mNZfnAFcAs0jrLU8Fynj8UUt2A/4iaR7pxqP5wC4RsRAgIh4lPadzBGn6/BfAQRHxSC6fnevYh7SW81Lg8Eq5mZmZtV9DU1NT23uZWU0azl1Y1z+opuM92WBmZsukhlp28ginmZmZmZXKwyq2TJF0MXBgC8VbFv+EpZmZmS0dPKVu1oHGjx/f1NjY2NndMDMzW1I8pW5mZmZmnc+B08zMzMxK5cBpZmZmZqVy4DQzMzOzUjlwmpmZmVmpHDjNzMzMrFQOnGZmZmZWKj+H06wDtfSnLf2nK83MrIvyczjNzMzMrPM5cJqZmZlZqRw4zczMzKxUDpxmZmZmVioHTusUknaUtLCz+1ErSQsl7djZ/TAzM1sWOXAuRyQNlvRiZ/djaSJpsqQDO7sfZmZmXZkD5xImaaXlqV0zMzMzPxywA0iaDPwaGAB8DngeOCIi/ihpDLAS8C7wDeB64AhJOwDnAFsA04FRETE617cjcB/wXeAMoAdwG3BURMzN+6ydjx8ArALcD3w/Il6r6tPXgS8CPwF+DKwsaW7u+h7ACOC6iBhVOJ8zgK9GxM6tnPNIYLOI+GZh247AeGB9oAkYB3wFWBV4ERgaEfe2UN8YYGFEDKm6rqdExLj8fjtgOLAlMAv4JfCziGj1YbKS9s3n3huYD9wVEQdJGg9sBFwm6WLg4YgYIKkHcCHQCLwNnNZa/WZmZtY6j3B2nMOBY4C1gJuAOyX1zGX7ABOAXsD/SuoP3AX8ClgbGAwMl7RPob4VSIHns8D/Az4J/AxAUgPwW1Ko+zTQlxSMrqnq06HAcaTAen7u48sR0T1/TQRGk4Itue5uwMHApW2c7xXAbpJ6FbYdDNwQEfNIP1u3AJvlc7wWuLlq/5pJ2hK4ExhJuo67A0cB32njuFWBscCREdED2Bi4DCAiGoGpwJB8PQbkw87P/d6SdP33JH0eZmZmVgePcHacyyPiCQBJI4DvkUYQAR6KiOvz6/mS9gOejIgxedujkkYDQ4AbC3UOjYg5wBxJpwG3Szoc+DzwBWCXiPh3bvOHwBuSekfEK/n4SyPiqfz6HUnN9fs6YJSkL0fEo8B/Ah8HftPayUbEXyU9BRyYj+8BfJs04koeiR1XOGSkpKGk0dY7W6u7Bd8DboyIW/P75yVdCAwCrmrj2HeBLST9KSLeAh5sacccuA8Ado+IGXnbUOBbdfTZzMzMcODsSJMrLyKiSdJU0hTuB8qyPsCkqm0vkUbSiqZU1f8xYB2gf379WlWI/BdpiviVwjGtioj5ksaRwu6j+ftVlSDbhiuAI4BRwH8Dr0TE/wFI+jhpNHK33OfFpJHWukY4See8k6S9Ctu6AdNaOyif326kkd6fSHoZOC8iqkeDK3qRru3kwrbqz8rMzMzawYGz4/SrvMhT3pXgtyUpbBVNIwWxoo35cHjqSwqilfr/DbxBCqLzgLUiorruouqylvYdDfyfpJ+QpvE/10qdRZXR0c+TlgVcUSg7Dtge2BmYnEP4G7T8N1ffJgVTACStCKxbKJ8C/Doijqyxb+/JSwcmSlqBtI72ZkmPRcRLfPiavAEsIF3v4rU3MzOzOnkNZ8c5RNLn893gJ5BulLmjhX2vBb4gaZCkFSV9CTgMuLxqv+GSekpaFxgGjM0BM4CngQvyzUNI6pVvjmnNDGDdwtpSACLiGeBZ4Gbg8Yj4ay0nHBGzSVPvZwFfBq4sFPckBeQ3STcqnQas0Up1TwA7S+ov6WOkm5yKd9b/EthXUqOklfJ12zLffNUiSetJ2lvS6hGxCJidixbl7zNI6zUr57SItBb29HxsT+CnrbVhZmZmrXPg7DiXABeQ7p4eSFoDOKe5HSNiEmmE8yhSIBsLnBoRNxR2W0QKrH8G/ga8TBo1JIfOPUmjhU9Ieps0Hb5jG328H7gXmCRpdlVYGw1sTds3C1W7AtgVuDsiphe2/4wU7v5BGimcT+tT/FeT7sR/Mu8/FXi1UhgRfyGtiT2WdFf/68AY2p6i7wYcCUzO1+ki4KCIqPTlLOBASbMkTcjbjiFNoz9Puv7jeT+gmpmZWTs1NDW1+kQZq0H143s6oL4dgfsiYoktecht/hbYICLmL6l2u5qGcxc2+w+q6XivXjEzsy6ppaVyH+ARTkPSKsDxpLvaHTbNzMysQ3nYZTmX7/oeR5rK/klV2QRgu+aOi4ju5feuNpJOAk5qoXjXiGjxMUhmZmZWPk+pm3UgT6mbmdlypqYpdf9f0KwD3bb5BBobGzu7G2ZmZksVr+E0MzMzs1I5cJqZmZlZqRw4zczMzKxUDpxmZmZmVioHTjMzMzMrlQOnmZmZmZXKgdPMzMzMSuUHv5t1oOoHv/uB72Zm1sX5b6mbmZmZWedz4DQzMzOzUjlwmpmZmVmpHDjNzMzMrFS+o8GWGEknAj8AVgO+HhF/XELtXgwsjIijlkR7ZmZm9kG+S92WCEm9ganApyPir53dn7L4LnUzM1vO+C51W6r0AxZ/lLApaaWO646ZmZktKR7htA+Q1B0YBuwF9AKmAYcBTwDD8/aPAw8BR0fE1HzcxLxPP2AA8DpwXETcKmkgMAZYBZgHvBYRm0haGxiV9we4G/hBRLyV65wM/Br4OvBFYAiwBbAdEMAhpF+afgLcDFyR93sBODAinsv1jCFNqQ/J75uAI4GDc33PAoMj4vlc3gO4CNgDeBs4Nfdjl4iY2Nr18winmZktZzzCaXW5HNgG2BnoCXwDmE4Khl/OX32BN4DxklYoHHsQcB6wOnAhcKWkVSPiemBXYFFEdI+ITfL+VwNrAv8vf60DjK3qz6HAcUAP4Na8bXvg78D6wIHAyNzvI4G1gOeAC9o4z8HA3rnNacAvCmU/BzYmhdHPALsDK2BmZmZ18fCLvUfSusB/k9ZZTsqbX5TUjRQmGyPi1bzvscBbwJeAR/K+10fEw7n8EuBnwGbA0820tQHwn8AnI2JW3nYc8LykT0TE9LzrpRHxVH79jiSAFyLisrxtgqQ3gbsLI5rXkMJsa0YWRmfHAOPy6xWAA4BdI+L1vO2kfF3MzMysDh7htKJ++fsLVdt7AR8DKiGUiJhLmjbvU9hveqF8Xn7Zo4W2KsdNKmx7qaoMYHIzx06vej+/atv8Vtptro55hf3XAVYGphTKi6/NzMysnRw4rWhy/r5Z1faZwL95P5BW1nquS5qOrkfluH6FbRtXlQEsrrP+er0BLCAtG6jYaAn3wczMrEvxlLq9JyJel3QT8EtJg0kje5X1llcBZ0r6KzCbtFbzeeDxOtv6h6R7gPMkHURadHweMKEwnb7ERcSiPCU/TNKfgX+RbkoyMzOzOnmE06odAvwJeIB0h/atpJtzfkC6M/yPpOdpfgL4RkQs+ghtHZjb+BspvM4GBn2E+jrKMaRzfAH4C3Av0EQa5TUzM7N28mORzNogaXNSIN4wIv7R2r5+LJKZmS1nanoskv9vaFZF0sakUd3HSDcRjQL+0FbYNDMzs+Z5St3sw1YBLgHmAH8m3fW+f6f2yMzMbBnmEU6zKvnPb366s/thZmbWVThwmnWg2zafQGNjY2d3w8zMbKniKXUzMzMzK5UDp5mZmZmVyoHTzMzMzErlwGlmZmZmpXLgNDMzM7NSOXCamZmZWakcOM3MzMysVA6cZmZmZlYqB04zMzMzK5UDp5mZmZmVyoHTzMzMzErlwGlmZmZmpXLgNDMzM7NSLVOBU1JvSU2S+tWw7zBJ9y2BbtVF0sWSLuzsfizNlqZrJOk+ScM6ux9mZmbLohU7uwOdJYfWSUCfiHilhv0HA6dExKYd0X5EHN4R9XQVkiaTru+4yrYlcY0kjQEWRsSQstsyMzNbXi1TI5y27JG0Umf3wczMzDrXUj3CKWl94BJgB+A14Jyq8kOBY4A+wMvA0Ii4p7BLg6RRwCDgHeDCiPhpLns6f/+bpCZgRESc2UI/tgUuBlaWNDdv3iMiJkraIfdrC2A6MCoiRtdwbmMojKzlPhwJHJzrehYYHBHP5/KVgBOAg4ANgNfz+d4kaUXgJGAwsCbwJHBMRPyl0NYKwLvAXsA84HjgOeDS3F4AB0TEP/IxqwJnAHsDqwOPA0dFxIttnNeOwH35PE4HegE9JB0DHAFsCMwCriaNaC6SNB7YCLhM0sXAwxExoJlr1Be4APgq6fO8GfhRRLzTRp+2Bn4BfAZYBDwP7A4cChyQ99k37746sBg4kfR5rApcCTS01oaZmZm1bGkf4byaFBA2ArYnBSrgvbA5lBQY1gROBm6RVJzy3p4UVD8B7AkcJ2n/XLZV/r55RHRvKWwCRMQjwOHAy3nf7jls9gfuAn4FrJ37N1zSPnWe72BSwFsHmEYKSRVnAQcC+wA9SSH8hVx2AilU7wasDzwI3CupZ+H4b5MC2lrAmaSgeQbwLWA9oIkUECsqQfTLuc7HgNtrHLFcIfdl61w3wCvArrnvewKHAEMAIqIRmAoMydd2QHWFOVTfAcwA+uZ+fRU4t4b+XATck899PeA4YEFEnEP6Gbuy8LkuIl3nH+R+rg+8QfpZMjMzszostSOckjYEdgI2jYg5wBxJp5OCA6SRzTMiojJSeaek+4F9SeEM0ojjiIhoAp6QdAkp1F3TQd3cD3gyIsbk949KGk0KUjfWUd/IiJgK741KjsuvG0ijbQMj4pm87yv5C9Jo4ojCaOgZuQ+7A9fmfX4fEXfk8qtIIXlsZf2qpJtII35IWgfYH+gbEa/lbacDxwLbAA/VcC5D8+cGQETcXCh7StJYYGegzdHg7EvAZsA2ETEPmCfpFOC3ko7Kn3FLFpB+aekTEZOBR9toaxAwOiKeAJA0nPQLh5mZmdVhqQ2cQO/8fUph26TC6/7ARZIuKGxbkfdDGMCUqiAymTSl3FH6VPUJ4CXSyFg9phdezwN65Ne9gNV4f0Sz1X5ExOJ8E06f5uqOiPmSqtubX2ivf/7+TN6vYqWqOluymDRC+x5J+5FGFjcmfU4r03bwK+oDzMxhs+IlYBXS9Xm9lWMPBk4FHpL0LinInx4RC1vYvzfpZwV473pOaWFfMzMza8PSHDhfzd/7koIFQL9C+RTgxxHR2khiX0kNhdDZj/cD6eJ29qe5/aeRpo6LNqYqbHWAmaRAuBnw9xb60a/yRlK3/L7eflTC1WYRMbOO45uKQV9SH1LI2wuYEBELJJ0LFNNsW5/HNKCXpFUjYn7etjHwL9L1aVFETCJN4SPpM6RR8kn8//buPFyuoszj+PclC1sIoCSKhJBEwGUAlRQ+OLIElE1RUYkIAiKKiSMqMqJOIJgRZBNkZIYMe8KiIDJDmIASFsnD8owwJajsgZAQCMQESGICCgRq/nirybkn3X373nT3TSe/z/P0c2+ftU6dOue8XUs3XFZjv/Ppmp+Gl0MRERHphTU24IwxPhtCmAmcFUL4CrAhcHJhkXOBSSGEJ/ABQBsAo4EXKk3LeN/NE/LAoR3wJuPj87xFeLCxHV1rRWtZAAwNIQyOMf41T7samBhCOBJvpt8ZGIcPjmmaGGMKIUzG82IePqBoK+BtuYl9KvD9EMKdeM3cD/Bze1Mv97cwhPBLYHII4bgY4/wQwmbAXsCtMcbl3WyibBDeX3gR8HoIYVfgCHzQUsUC/FzUch/wJHBOCOGfgc3wvqhTumlOJ4Tw5Zzu54AlwAq8b3Blv7uGENaLMVaCzyvxvL4eeBAfYPXORg9WREREulrTBw0dBqyP127dBVxRmRFjvBgfHT4FH/U8D282LQ5quQsPOhcANwI/J/ffzCObJwJXhxCWhBBO7CYtdwC3AnPy8nvmmrNPAMcCL+KBysQY47Wrc9A1nAhcC0wDlgEzgcoAqZ/iwe8t+CCpvYF9C4FxbxwDPA7MDCEswwOvsfjgoh6JMT4K/Ai4AQ/4fsjKvqUVpwKHhxAWhxB+W2UbK4AD8ebueXgAei8eDHZnb7wP78vA/+Jl4Mo87xK8u8KL+bz2w8vZvwPT8fwcCtzZ8AGLiIhIF5ZSj+MHEalh+vTp6VOf+lRfJ0NERKRdGvrawDW9hlNEREREOtwa24ez3UIIuwOrNOVmp8UYT+vh9ibgX8ZezQExxrt6sr01SeHL78vuijEe0NbE0PxzJyIiIs2lJnWRJlKTuoiIrGPUpC4iIiIifU8Bp4iIiIi0lAJOEREREWkpBZwiIiIi0lIKOEVERESkpRRwioiIiEhLKeAUERERkZZSwCkiIiIiLaWAU0RERERaSgGniIiIiLSUAk4RERERaSkFnCIiIiLSUgo4RURERKSlFHCKiIiISEsp4BQRERGRllLAKSIiIiItpYBTRERERFrKUkp9nQaRtcb666//0Guvvfb3vk7H2qp///5brFix4oW+TsfaSvnbWsrf1lL+tlad/H0hpbR/t+u3IE0i66wdd9zx7zHG0NfpWFuFEKLyt3WUv62l/G0t5W9rrW7+qkldRERERFpKAaeIiIiItJQCTpHmuqivE7CWU/62lvK3tZS/raX8ba3Vyl8NGhIRERGRllINp4iIiIi0lAJOEREREWkpfS2SSA+FELYHLgfeDrwIHBljfKK0TD/gPGB/IAFnxBgvaXdaO1GD+TsR+CLwBvA6MCHGOKPdae1EjeRvYdn3AA8Ak2OM32tfKjtXo/kbQvgCMBEw/B7x8RjjX9qZ1k7U4P1hKDAF2BoYANwBfDvGuKLNye04IYSzgc8DI4AdY4wPVVmmV8831XCK9NwFwPkxxu2B84ELqyzzJWBbYDvgI8CkEMKItqWwszWSv/cBu8QYdwKOBn4VQtiwjWnsZI3kb+WhciEwrY1pWxt0m78hhABMAvaJMe4A7AYsbWciO1gj5XcC8Gi+P+wEjAY+174kdrRpwB7A03WW6dXzTQGnSA/kT847A1fnSVcDO4cQhpQWPQS4OMb4ZoxxEX4Rj21fSjtTo/kbY5wRY3wlv/0zXkv09rYltEP1oPwC/BC4EZjVpuR1vB7k73eBs2OMCwBijEtjjPqFsm70IH8TsEkIYT1gfWAgML9tCe1gMca7Y4zPdLNYr55vCjhFemZrYH6M8Q2A/Pe5PL1oOF0/Ic6rsoysqtH8LToSmB1jfLYN6et0DeVvCOEDwH7AuW1PYWdrtPy+HxgVQrgzhHB/COGkEIK1Oa2dqNH8PQXYHngeWADMiDHe086EruV69XxTwCkiHSuEsCf+cDm0r9OytgghDMC/b2985cEuTdcPb+rdB9gTOAA4ok9TtHYZi7d8bAlsBewRQji4b5MkCjhFeuYZYKvcv63Sz+1deXrRPGCbwvvhVZaRVTWav4QQPgJcBRwUY3y8ransXI3k75bAu4HfhBDmAscBx4QQ9KXa3evJ/eG6GOOrMcZlwA3Ah9ua0s7UaP5+C/hFbvJdiufvXm1N6dqtV883BZwiPRBjXAj8kZU1aocCD+R+LEW/xh/S6+X+RQcB17UvpZ2p0fwNIewC/Ao4OMZ4f3tT2bkayd8Y47wY4xYxxhExxhHAv+H9tb7e9gR3mB7cH34J7BtCsFyj/DHgT+1LaWfqQf7OwUdQE0IYCHwcWGW0tfRar55vCjhFem488K0Qwiz8k/R4gBDCb/LoU4ArgaeAJ4DfAz+OMc7pi8R2oEbydzKwIXBhCOGP+bVj3yS34zSSv9J7jeTvNcBC4BE8gHoYuLQP0tqJGsnf44DdQwgP4vk7C7i4LxLbaUII54UQngWGAbeFEB7O01f7+aafthQRERGRllINp4iIiIi0lAJOEREREWkpBZwiIiIi0lIKOEVERESkpRRwioiIiEhLKeAUWYeZ2X5mdlfh/Rgzm9uHSWobM5tqZpc0cXsjzCwV3g8xs6fNbIsG1h1vZlc2Ky2dwMx2N7MlfZ2OdZGZHd6T67zZ14rU16proxfn/QwzO6VZ+1fAKbKOMjPDfyv7R90s9w0ze8jM/mpmi80smtkhhflzzezwKuutMt3crLytQaV5Y8wsmdny/HrOzKaY2dtW70j7RkppEf4F393l78bAj4FJbUjWGiOldFdKabO+TkctZjbJzG7r63SsC1qV12Y208xOavZ2W618bfRhWTwT+KaZbdWMjSngFFl37QsMBO6otYCZHYoHTF8FNsV/Ru67wOJe7nMvYBTwJtV///yNlNKglNIgYDfgI/gv3XSqy4CvmNngOsscDjyYUprdpjR1YWb9zEzPAhHpIqW0GPgtMK4Z29NNRqQNcm3fSWZ2R669e9DMdjKzQ83sSTNbamaXmFn/wjrDzew6M1tgZs+b2UVmtklh/mlm9lTe3mwzO64wb0SuLTzCzB4xs2VmdouZbVlI1kHAban+rz/8I3BnSune5P6WP33f0susGAfcjP9SRd2bWErpKeBG4EPleWbWP+fJQaXpU81sSv7/Y2Z2b66VXWRm15jZ0Fr7y/m1W+H9GDNbUdrnhFxDu8TM7jGzur/Mk1J6AsGX2X8AAAvzSURBVHgB/2m9Wg4Cbi2l5Ttm9lg+b/PM7HQz65fn/dTMppWWH5OX3Ti/38HMZuTjrqw/IM+rlI2vmtkjwCvAUDP7opn9Kdc+P29mF1a2l9d7p5lNz2V1Vl4/mdmIwjLH5NrwpWb2gJntW+ugq+TvVDO70swuy/k7P18fHzSz/8vHd4eZvauwzlwzO9nM7s7XQTSzXQrz65YBMxuQz+njefuzzexg8xr8CcAYW1njPqrGceyZ97E0n7NxhXljzGyFmR2St73UzK4tXsdVttebe8VOZva7fJxP5fX7FeZ/OOfNcjO7G//QV9znRmZ2tpnNMbOXzOxmM9u2VhqrpPntZnaF+b1qgZldboWWCSu1dhTK4LBaeW1mR+Xj/UEujwvN7Jwq5XhYYbtHmdmT+f//AHYHJuZtPl4j7ZPM7HYzOzOXkRfN7Hgz2ybn6TIz+4OZva+wzmpdK4WyfnGhrK9SbvL/dfOndCxduj406bzfit+jVl9KSS+99GrxC5iL/wzY+4ABwFXAbOAiYGNgOP5Td1/Ky28APIk3tW4IbA78BrissM3D8RpHA/YG/gbsl+eNABIesG0BDAbuAS4urH8v8O1SOscAcwvvxwJ/B07Ff+95sxrHdnh304EhwKvA5/AgMgGjS/teUXi/LfB48ZhL2z8LmFZ4PwhYDuye3+8G7AL0B94J3AlcXVh+KnBJ4X0CdquTnp/kPBsF9MNrfV8ANi/meZV0TgdOrVM2/gJ8ujTt88DIfG4/lJcZl+e9H3gNGFJY/nLg0vz/UOBFPKAfCGwFRODkUtm4PefLwHw8BwD/gFdEbIv/7OLphX3cDvxXLktDgZl5OyPy/GPwMvuBvI1P5POxbY3jLufvVLwMfzKvPz6v/z/4z+xtBPyOrmV4LvAcMDofxw+BRcDgBsvAmfk4d8p5PQzYKc+bhH8gq3ddj8xpPirvY1fgJWBs4RgT/rOVg4B34PeBE5t4r9g0l4+JwPp5vaeAEwrzX8x5MzDnxwK6Xue/wO8V78jL/CvwGDCg2rVSJc034+V88/y6Cbipzr1gRM6XYbXyOufp68D5+D3w3fhPVE6oto3COk8W3s8ETurmHE7K+/kaK6+DN4DbSufg1sI6q3utTMXLzafzNj6X07BNjWujVv48WZr21nlqxnnPy4zGW6QG1svHRl5tfejqpde6+so33BMK7z+Rb0DFoOFa4Nz8/8HA7NI2RuMBW78a+7gOOCv/X7kZ71KY/03ggcL7WcBRpW2MKd6Q8rQDgf/GH2pv4E3wO5SO7WVgSen1Jl0fMt/HH5SVh9j9wIWlfae87mJgDnABVYLcvPz78MBraH5/NDCrzjk4EFhYeP/WzTm/rxlw4sHIMmCP0jYfrBwjtQPOXwCT66TrNWBMN+XnbODawvt7ge/m/zfJ+f/R/P57wO9K63+e/HAqlI09utnnscB9+f9heZ1Rhfkfo+tD9CHgyNI2plPjgU/1gLMYpGyUtz+2MO2f6FqG5wKnFN4bMA84rLsykJddDnyyxrKT6D7gnADcU5p2OjCjVKaL1/lPgevrbHMuPbtXHAY8Q/6p6jxtHPB4/v9LOU+K839Cvs7xD6QJGF6Yvx6wlHw9UCfgxD/0JmC7wrT35GlbFo6pNwHnq8BGhWlfI1/j5W0U1ulNwPlwadrCKudgcROvlakUynqetgj4TI1ro1b+1As4V/u852nb5eWG1svHRl5vVcmLSMs9X/j/Fby/4qLStEpT20hguK06UjHhNTXzzezbeK3SMPzhuSE+SKXWPl8ubB88qKvXt9B3mNKN+KdgzOy9wGTgRjMbmfIdCa99u6q4nhVGQ5qZ5bRelVJ6PU++FDjDzL6XUlqWp72RGhxIklJ61Mzux2t6fwZ8BZhS2Odo4DS8xm0jPI8GVdlUI7bI6063wkh0vPZjWPVV3jIYD55rWeU8mPedPR6vTe2P1z78vrDIFOAb+KCvLwDPppTuyfNGAh8tlR3Da2+K5pb2uQ9wMvBevKasH/7gBa8lBX+AVTxd2t5I4HwzO68wrT/wLI17q7ymlF7xYrPKdVNujp5bWCeZ2TzyOemmDAzBawxn9SB9ZVuz6rmdDXym8L58nZevw2p6cq/YGni6cC1W0rB1/n9YlfnFNI/Mf/+c87tiQGEb9VSWKW5zdmHe8/TewpTSK4X3c+n+euuNchpfoU65a8K1Um2fjZSLnmjWeR/MyoqA1aI+nCJrpqfxT/KblV4bpJTmm9lH8ebAccAWOUibjj9QG/UA3jzbsJTSY3iQsw3edNaovfGmp6Mr/bzw5ptBeA1Nb00Bjsr9jnYFrijMuwavRd0+pTSY6oOUipbjAUjFuwr/v4A/ED5eOh8bp5TO6Ga7O+B5XUuX82BmW+NNeKfiNUSb4s2KxXN7DbC9me2M13RMKcx7Gq8NKaZz0+QDsYreLOxzIDAtb3d4zq8fFPY5P/8dXli/+H9lv0eX9jsopfSNOsfeDCMq/+QPNsNZGeTWKwOL8EBiuxrbfbPG9KJnivvPRuXp7fIMsI11jRqKaZhfZf6Iwv+VYGi70rnbKKV0dYP7L29zVGneMmpfW1A7r4ea2UaldFfObeVDam+222tNulZ6qtpxlPMUuh5/s877DngN8Gu9TPtbFHCKrJluBAaaD2jYxNxWZvbZPH8w3ry9CEhm9km8X1FPTMObemoys6PNbKzl75LMHfTHA4+klF7qwb7G4f3n3gt8ML92wAOlr/cw3UXX4IHseXgfq/mFeYPx5qFlZjYc78tUzx+AL5vZwNy5//jKjFxL8HPgbDPbDsDMBpl/j2n5IfeWHAgPwfuD1TKNroOKBuH35kXA62a2K3BEcYWU0hLgejwo3RXvw1lxBRDyudvAzNbLgwz2r5OGgXhNzeKU0t/M7P14M2Flf8/izZNn5PI4BCh/3cy5wCTzQT5mZhua2W65VryVjjaznc0Hk5yA12TelOfVLAP5nE4GzjIfZGXmg1h2yosswFsZBtbZ99XAaDM70nxQ2Yfxsn5pU4+wvpvwczchl9334AFQJQ034mXqBPNBUjvj/Y8BSCktxFtGJlv++hsz28zMPmulry6rJqX0HHALcE5eb3PgHOC3KaVKLd4fgEPzNTME729aVCuv1wPOzGVpFN5d5PK83xfJH3LMv2lhR7wVpbzdhgc/NagZ10pPVcufP+IB+YH5Gv8ssEdhfrPO+z74PWq1KeAUWQPlZqS98Zqvx/CH5u14oAYwAw8s7sNr3w7GA5CemAGsMLMxdZZZjDfdPmpmL+N9B5fgfeEaYj4q+CDg7JTSguILr6X9kHUz2ruWlNJS/LgPwL+CqOjreJ+vZXgf1F93s7lj8YfTS3gfuaml+T8CbgBuMLO/4gM7xlP/Pno0MDWns5YrgQ/kByoppUcL+1qCB0nVapqm4Mc9o/BgJ+frXniez8XP4fWURqgWpZSW4+f5LDNbjteolrtnHIYHc8/iA9Aq+flq3sbF+ECuKXmf8/DAYkCdY2+Gi/APHIuBQ/A+mZX87q4MnIif62l5mZmsDFB+jdfQLTAfSTyytC4ppTl4/75j8QEaVwITU0rXNuvgupOPdV/8Q8tfWHlv+FmevwQfiHUInkfnAf9Z2swx+AC9mWa2DO+bPBZvSm3E4Xj+PY7fr5YARxbmn4R/QH4ez+NrSuvXyuun8fI2B7/33IyXsYov4/eipfl4y4H+ufiHryVm9nCDx1JXM66VXlglf5J/jdp38PL/ErA/PlCpks7VPu9mthlevi/oZbq7sK7N+yKyLsm1XhNSSnvk92PwAGlEX6arE+Va0TkpJcvvh+Cjw0Op/121dcfjg36OqLfcmsTM9sOD4g1THz1IzPsJn1TuPyydz8yOws9ts2so225NuFZ6w8xOx/sPN+XL8zVoSGQdllK6Ga81kCbLQeY2DS57AU2qRWgVM/sg3pfsQXzAwanArzrpASrSDmvLtZJS+pdmbk9N6iJSNJfO/mWfvrQEHwi1ttocb5ZeDtwN/Blv0hORrnStVKEmdRERERFpKdVwioiIiEhLKeAUERERkZZSwCkiIiIiLaWAU0RERERaSgGniIiIiLTU/wONnE8ghVgg9wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 576x396 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "shap.summary_plot(shap_values_gbm_default, X, plot_type=\"bar\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build Default XGBoost with Monotonic Constraints"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "xgboost Model Build progress: |███████████████████████████████████████████| 100%\n"
     ]
    }
   ],
   "source": [
    "mxgb_default = H2OXGBoostEstimator(\n",
    "               # monotonicity constraints\n",
    "               monotone_constraints=mono_constraints, \n",
    "               seed=SEED)\n",
    "mxgb_default.train(x=X,\n",
    "              y=Y,\n",
    "              training_frame=train,\n",
    "              fold_column=fold_column)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model Summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model Details\n",
      "=============\n",
      "H2OXGBoostEstimator :  XGBoost\n",
      "Model Key:  XGBoost_model_python_1575925687897_931\n",
      "\n",
      "\n",
      "Model Summary: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>number_of_trees</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td></td>\n",
       "      <td>50.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     number_of_trees\n",
       "0               50.0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "ModelMetricsBinomial: xgboost\n",
      "** Reported on train data. **\n",
      "\n",
      "MSE: 0.07762546454250034\n",
      "RMSE: 0.27861346798476977\n",
      "LogLoss: 0.2600707538323361\n",
      "Mean Per-Class Error: 0.2576287073263952\n",
      "AUC: 0.8062379074746303\n",
      "pr_auc: 0.26888416210329724\n",
      "Gini: 0.6124758149492606\n",
      "\n",
      "Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.1690960709424414: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>Error</th>\n",
       "      <th>Rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>114157.0</td>\n",
       "      <td>30676.0</td>\n",
       "      <td>0.2118</td>\n",
       "      <td>(30676.0/144833.0)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>5772.0</td>\n",
       "      <td>9733.0</td>\n",
       "      <td>0.3723</td>\n",
       "      <td>(5772.0/15505.0)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Total</td>\n",
       "      <td>119929.0</td>\n",
       "      <td>40409.0</td>\n",
       "      <td>0.2273</td>\n",
       "      <td>(36448.0/160338.0)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 0        1   Error                 Rate\n",
       "0      0  114157.0  30676.0  0.2118   (30676.0/144833.0)\n",
       "1      1    5772.0   9733.0  0.3723     (5772.0/15505.0)\n",
       "2  Total  119929.0  40409.0  0.2273   (36448.0/160338.0)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Maximum Metrics: Maximum metrics at their respective thresholds\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>metric</th>\n",
       "      <th>threshold</th>\n",
       "      <th>value</th>\n",
       "      <th>idx</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>max f1</td>\n",
       "      <td>0.169096</td>\n",
       "      <td>0.348142</td>\n",
       "      <td>163.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>max f2</td>\n",
       "      <td>0.104239</td>\n",
       "      <td>0.514039</td>\n",
       "      <td>237.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>max f0point5</td>\n",
       "      <td>0.233018</td>\n",
       "      <td>0.293056</td>\n",
       "      <td>105.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>max accuracy</td>\n",
       "      <td>0.474129</td>\n",
       "      <td>0.903523</td>\n",
       "      <td>31.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>max precision</td>\n",
       "      <td>0.875909</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>max recall</td>\n",
       "      <td>0.001659</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>395.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>max specificity</td>\n",
       "      <td>0.875909</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>max absolute_mcc</td>\n",
       "      <td>0.126256</td>\n",
       "      <td>0.292397</td>\n",
       "      <td>210.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>max min_per_class_accuracy</td>\n",
       "      <td>0.145369</td>\n",
       "      <td>0.726540</td>\n",
       "      <td>190.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>max mean_per_class_accuracy</td>\n",
       "      <td>0.097395</td>\n",
       "      <td>0.742371</td>\n",
       "      <td>245.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>max tns</td>\n",
       "      <td>0.875909</td>\n",
       "      <td>144833.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>max fns</td>\n",
       "      <td>0.875909</td>\n",
       "      <td>15504.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>max fps</td>\n",
       "      <td>0.000294</td>\n",
       "      <td>144833.000000</td>\n",
       "      <td>399.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>max tps</td>\n",
       "      <td>0.001659</td>\n",
       "      <td>15505.000000</td>\n",
       "      <td>395.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>max tnr</td>\n",
       "      <td>0.875909</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>max fnr</td>\n",
       "      <td>0.875909</td>\n",
       "      <td>0.999936</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>max fpr</td>\n",
       "      <td>0.000294</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>399.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>max tpr</td>\n",
       "      <td>0.001659</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>395.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         metric  threshold          value    idx\n",
       "0                        max f1   0.169096       0.348142  163.0\n",
       "1                        max f2   0.104239       0.514039  237.0\n",
       "2                  max f0point5   0.233018       0.293056  105.0\n",
       "3                  max accuracy   0.474129       0.903523   31.0\n",
       "4                 max precision   0.875909       1.000000    0.0\n",
       "5                    max recall   0.001659       1.000000  395.0\n",
       "6               max specificity   0.875909       1.000000    0.0\n",
       "7              max absolute_mcc   0.126256       0.292397  210.0\n",
       "8    max min_per_class_accuracy   0.145369       0.726540  190.0\n",
       "9   max mean_per_class_accuracy   0.097395       0.742371  245.0\n",
       "10                      max tns   0.875909  144833.000000    0.0\n",
       "11                      max fns   0.875909   15504.000000    0.0\n",
       "12                      max fps   0.000294  144833.000000  399.0\n",
       "13                      max tps   0.001659   15505.000000  395.0\n",
       "14                      max tnr   0.875909       1.000000    0.0\n",
       "15                      max fnr   0.875909       0.999936    0.0\n",
       "16                      max fpr   0.000294       1.000000  399.0\n",
       "17                      max tpr   0.001659       1.000000  395.0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Gains/Lift Table: Avg response rate:  9.67 %, avg score:  9.74 %\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>group</th>\n",
       "      <th>cumulative_data_fraction</th>\n",
       "      <th>lower_threshold</th>\n",
       "      <th>lift</th>\n",
       "      <th>cumulative_lift</th>\n",
       "      <th>response_rate</th>\n",
       "      <th>score</th>\n",
       "      <th>cumulative_response_rate</th>\n",
       "      <th>cumulative_score</th>\n",
       "      <th>capture_rate</th>\n",
       "      <th>cumulative_capture_rate</th>\n",
       "      <th>gain</th>\n",
       "      <th>cumulative_gain</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "      <td>0.011064</td>\n",
       "      <td>0.356938</td>\n",
       "      <td>4.226191</td>\n",
       "      <td>4.226191</td>\n",
       "      <td>0.408681</td>\n",
       "      <td>0.411933</td>\n",
       "      <td>0.408681</td>\n",
       "      <td>0.411933</td>\n",
       "      <td>0.046759</td>\n",
       "      <td>0.046759</td>\n",
       "      <td>322.619063</td>\n",
       "      <td>322.619063</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td></td>\n",
       "      <td>2</td>\n",
       "      <td>0.020588</td>\n",
       "      <td>0.331251</td>\n",
       "      <td>3.690814</td>\n",
       "      <td>3.978532</td>\n",
       "      <td>0.356909</td>\n",
       "      <td>0.337290</td>\n",
       "      <td>0.384732</td>\n",
       "      <td>0.377404</td>\n",
       "      <td>0.035150</td>\n",
       "      <td>0.081909</td>\n",
       "      <td>269.081398</td>\n",
       "      <td>297.853230</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td></td>\n",
       "      <td>3</td>\n",
       "      <td>0.032257</td>\n",
       "      <td>0.300754</td>\n",
       "      <td>3.089603</td>\n",
       "      <td>3.656957</td>\n",
       "      <td>0.298771</td>\n",
       "      <td>0.316449</td>\n",
       "      <td>0.353635</td>\n",
       "      <td>0.355353</td>\n",
       "      <td>0.036053</td>\n",
       "      <td>0.117962</td>\n",
       "      <td>208.960324</td>\n",
       "      <td>265.695723</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td></td>\n",
       "      <td>4</td>\n",
       "      <td>0.040022</td>\n",
       "      <td>0.273410</td>\n",
       "      <td>3.156305</td>\n",
       "      <td>3.559823</td>\n",
       "      <td>0.305221</td>\n",
       "      <td>0.286516</td>\n",
       "      <td>0.344242</td>\n",
       "      <td>0.341998</td>\n",
       "      <td>0.024508</td>\n",
       "      <td>0.142470</td>\n",
       "      <td>215.630481</td>\n",
       "      <td>255.982270</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td></td>\n",
       "      <td>5</td>\n",
       "      <td>0.050300</td>\n",
       "      <td>0.261326</td>\n",
       "      <td>2.698211</td>\n",
       "      <td>3.383761</td>\n",
       "      <td>0.260922</td>\n",
       "      <td>0.265141</td>\n",
       "      <td>0.327216</td>\n",
       "      <td>0.326293</td>\n",
       "      <td>0.027733</td>\n",
       "      <td>0.170203</td>\n",
       "      <td>169.821119</td>\n",
       "      <td>238.376123</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td></td>\n",
       "      <td>6</td>\n",
       "      <td>0.100288</td>\n",
       "      <td>0.240890</td>\n",
       "      <td>2.623002</td>\n",
       "      <td>3.004564</td>\n",
       "      <td>0.253649</td>\n",
       "      <td>0.250942</td>\n",
       "      <td>0.290547</td>\n",
       "      <td>0.288734</td>\n",
       "      <td>0.131119</td>\n",
       "      <td>0.301322</td>\n",
       "      <td>162.300153</td>\n",
       "      <td>200.456415</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td></td>\n",
       "      <td>7</td>\n",
       "      <td>0.151199</td>\n",
       "      <td>0.213675</td>\n",
       "      <td>2.414559</td>\n",
       "      <td>2.805900</td>\n",
       "      <td>0.233493</td>\n",
       "      <td>0.223936</td>\n",
       "      <td>0.271336</td>\n",
       "      <td>0.266916</td>\n",
       "      <td>0.122928</td>\n",
       "      <td>0.424250</td>\n",
       "      <td>141.455883</td>\n",
       "      <td>180.590007</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td></td>\n",
       "      <td>8</td>\n",
       "      <td>0.200002</td>\n",
       "      <td>0.188826</td>\n",
       "      <td>2.111821</td>\n",
       "      <td>2.636536</td>\n",
       "      <td>0.204217</td>\n",
       "      <td>0.202337</td>\n",
       "      <td>0.254958</td>\n",
       "      <td>0.251158</td>\n",
       "      <td>0.103064</td>\n",
       "      <td>0.527314</td>\n",
       "      <td>111.182108</td>\n",
       "      <td>163.653596</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td></td>\n",
       "      <td>9</td>\n",
       "      <td>0.300191</td>\n",
       "      <td>0.151126</td>\n",
       "      <td>1.772218</td>\n",
       "      <td>2.348071</td>\n",
       "      <td>0.171377</td>\n",
       "      <td>0.169119</td>\n",
       "      <td>0.227063</td>\n",
       "      <td>0.223777</td>\n",
       "      <td>0.177556</td>\n",
       "      <td>0.704869</td>\n",
       "      <td>77.221826</td>\n",
       "      <td>134.807092</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td></td>\n",
       "      <td>10</td>\n",
       "      <td>0.400092</td>\n",
       "      <td>0.109434</td>\n",
       "      <td>1.300217</td>\n",
       "      <td>2.086426</td>\n",
       "      <td>0.125734</td>\n",
       "      <td>0.128625</td>\n",
       "      <td>0.201761</td>\n",
       "      <td>0.200018</td>\n",
       "      <td>0.129894</td>\n",
       "      <td>0.834763</td>\n",
       "      <td>30.021708</td>\n",
       "      <td>108.642598</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td></td>\n",
       "      <td>11</td>\n",
       "      <td>0.500025</td>\n",
       "      <td>0.068497</td>\n",
       "      <td>0.826742</td>\n",
       "      <td>1.834671</td>\n",
       "      <td>0.079948</td>\n",
       "      <td>0.087309</td>\n",
       "      <td>0.177416</td>\n",
       "      <td>0.177493</td>\n",
       "      <td>0.082619</td>\n",
       "      <td>0.917381</td>\n",
       "      <td>-17.325802</td>\n",
       "      <td>83.467144</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td></td>\n",
       "      <td>12</td>\n",
       "      <td>0.600026</td>\n",
       "      <td>0.033110</td>\n",
       "      <td>0.475970</td>\n",
       "      <td>1.608228</td>\n",
       "      <td>0.046027</td>\n",
       "      <td>0.048891</td>\n",
       "      <td>0.155519</td>\n",
       "      <td>0.156060</td>\n",
       "      <td>0.047598</td>\n",
       "      <td>0.964979</td>\n",
       "      <td>-52.403045</td>\n",
       "      <td>60.822819</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td></td>\n",
       "      <td>13</td>\n",
       "      <td>0.700071</td>\n",
       "      <td>0.012920</td>\n",
       "      <td>0.210805</td>\n",
       "      <td>1.408527</td>\n",
       "      <td>0.020385</td>\n",
       "      <td>0.021986</td>\n",
       "      <td>0.136207</td>\n",
       "      <td>0.136900</td>\n",
       "      <td>0.021090</td>\n",
       "      <td>0.986069</td>\n",
       "      <td>-78.919495</td>\n",
       "      <td>40.852695</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td></td>\n",
       "      <td>14</td>\n",
       "      <td>0.800147</td>\n",
       "      <td>0.006035</td>\n",
       "      <td>0.088291</td>\n",
       "      <td>1.243402</td>\n",
       "      <td>0.008538</td>\n",
       "      <td>0.008514</td>\n",
       "      <td>0.120239</td>\n",
       "      <td>0.120842</td>\n",
       "      <td>0.008836</td>\n",
       "      <td>0.994905</td>\n",
       "      <td>-91.170859</td>\n",
       "      <td>24.340232</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td></td>\n",
       "      <td>15</td>\n",
       "      <td>0.900011</td>\n",
       "      <td>0.003303</td>\n",
       "      <td>0.038750</td>\n",
       "      <td>1.109736</td>\n",
       "      <td>0.003747</td>\n",
       "      <td>0.004695</td>\n",
       "      <td>0.107314</td>\n",
       "      <td>0.107955</td>\n",
       "      <td>0.003870</td>\n",
       "      <td>0.998775</td>\n",
       "      <td>-96.125012</td>\n",
       "      <td>10.973570</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td></td>\n",
       "      <td>16</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000023</td>\n",
       "      <td>0.012255</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.001185</td>\n",
       "      <td>0.002075</td>\n",
       "      <td>0.096702</td>\n",
       "      <td>0.097368</td>\n",
       "      <td>0.001225</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-98.774451</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      group  cumulative_data_fraction  lower_threshold      lift  \\\n",
       "0         1                  0.011064         0.356938  4.226191   \n",
       "1         2                  0.020588         0.331251  3.690814   \n",
       "2         3                  0.032257         0.300754  3.089603   \n",
       "3         4                  0.040022         0.273410  3.156305   \n",
       "4         5                  0.050300         0.261326  2.698211   \n",
       "5         6                  0.100288         0.240890  2.623002   \n",
       "6         7                  0.151199         0.213675  2.414559   \n",
       "7         8                  0.200002         0.188826  2.111821   \n",
       "8         9                  0.300191         0.151126  1.772218   \n",
       "9        10                  0.400092         0.109434  1.300217   \n",
       "10       11                  0.500025         0.068497  0.826742   \n",
       "11       12                  0.600026         0.033110  0.475970   \n",
       "12       13                  0.700071         0.012920  0.210805   \n",
       "13       14                  0.800147         0.006035  0.088291   \n",
       "14       15                  0.900011         0.003303  0.038750   \n",
       "15       16                  1.000000         0.000023  0.012255   \n",
       "\n",
       "    cumulative_lift  response_rate     score  cumulative_response_rate  \\\n",
       "0          4.226191       0.408681  0.411933                  0.408681   \n",
       "1          3.978532       0.356909  0.337290                  0.384732   \n",
       "2          3.656957       0.298771  0.316449                  0.353635   \n",
       "3          3.559823       0.305221  0.286516                  0.344242   \n",
       "4          3.383761       0.260922  0.265141                  0.327216   \n",
       "5          3.004564       0.253649  0.250942                  0.290547   \n",
       "6          2.805900       0.233493  0.223936                  0.271336   \n",
       "7          2.636536       0.204217  0.202337                  0.254958   \n",
       "8          2.348071       0.171377  0.169119                  0.227063   \n",
       "9          2.086426       0.125734  0.128625                  0.201761   \n",
       "10         1.834671       0.079948  0.087309                  0.177416   \n",
       "11         1.608228       0.046027  0.048891                  0.155519   \n",
       "12         1.408527       0.020385  0.021986                  0.136207   \n",
       "13         1.243402       0.008538  0.008514                  0.120239   \n",
       "14         1.109736       0.003747  0.004695                  0.107314   \n",
       "15         1.000000       0.001185  0.002075                  0.096702   \n",
       "\n",
       "    cumulative_score  capture_rate  cumulative_capture_rate        gain  \\\n",
       "0           0.411933      0.046759                 0.046759  322.619063   \n",
       "1           0.377404      0.035150                 0.081909  269.081398   \n",
       "2           0.355353      0.036053                 0.117962  208.960324   \n",
       "3           0.341998      0.024508                 0.142470  215.630481   \n",
       "4           0.326293      0.027733                 0.170203  169.821119   \n",
       "5           0.288734      0.131119                 0.301322  162.300153   \n",
       "6           0.266916      0.122928                 0.424250  141.455883   \n",
       "7           0.251158      0.103064                 0.527314  111.182108   \n",
       "8           0.223777      0.177556                 0.704869   77.221826   \n",
       "9           0.200018      0.129894                 0.834763   30.021708   \n",
       "10          0.177493      0.082619                 0.917381  -17.325802   \n",
       "11          0.156060      0.047598                 0.964979  -52.403045   \n",
       "12          0.136900      0.021090                 0.986069  -78.919495   \n",
       "13          0.120842      0.008836                 0.994905  -91.170859   \n",
       "14          0.107955      0.003870                 0.998775  -96.125012   \n",
       "15          0.097368      0.001225                 1.000000  -98.774451   \n",
       "\n",
       "    cumulative_gain  \n",
       "0        322.619063  \n",
       "1        297.853230  \n",
       "2        265.695723  \n",
       "3        255.982270  \n",
       "4        238.376123  \n",
       "5        200.456415  \n",
       "6        180.590007  \n",
       "7        163.653596  \n",
       "8        134.807092  \n",
       "9        108.642598  \n",
       "10        83.467144  \n",
       "11        60.822819  \n",
       "12        40.852695  \n",
       "13        24.340232  \n",
       "14        10.973570  \n",
       "15         0.000000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "ModelMetricsBinomial: xgboost\n",
      "** Reported on cross-validation data. **\n",
      "\n",
      "MSE: 0.0782797307863066\n",
      "RMSE: 0.2797851511183297\n",
      "LogLoss: 0.26391161672737234\n",
      "Mean Per-Class Error: 0.2640070213704946\n",
      "AUC: 0.798732101317958\n",
      "pr_auc: 0.25492089548757935\n",
      "Gini: 0.597464202635916\n",
      "\n",
      "Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.16327585172085535: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>Error</th>\n",
       "      <th>Rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>112578.0</td>\n",
       "      <td>32255.0</td>\n",
       "      <td>0.2227</td>\n",
       "      <td>(32255.0/144833.0)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>5584.0</td>\n",
       "      <td>9921.0</td>\n",
       "      <td>0.3601</td>\n",
       "      <td>(5584.0/15505.0)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Total</td>\n",
       "      <td>118162.0</td>\n",
       "      <td>42176.0</td>\n",
       "      <td>0.236</td>\n",
       "      <td>(37839.0/160338.0)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 0        1   Error                 Rate\n",
       "0      0  112578.0  32255.0  0.2227   (32255.0/144833.0)\n",
       "1      1    5584.0   9921.0  0.3601     (5584.0/15505.0)\n",
       "2  Total  118162.0  42176.0   0.236   (37839.0/160338.0)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Maximum Metrics: Maximum metrics at their respective thresholds\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>metric</th>\n",
       "      <th>threshold</th>\n",
       "      <th>value</th>\n",
       "      <th>idx</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>max f1</td>\n",
       "      <td>0.163276</td>\n",
       "      <td>0.343995</td>\n",
       "      <td>189.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>max f2</td>\n",
       "      <td>0.106512</td>\n",
       "      <td>0.507641</td>\n",
       "      <td>252.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>max f0point5</td>\n",
       "      <td>0.215648</td>\n",
       "      <td>0.285605</td>\n",
       "      <td>140.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>max accuracy</td>\n",
       "      <td>0.952242</td>\n",
       "      <td>0.903304</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>max precision</td>\n",
       "      <td>0.952242</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>max recall</td>\n",
       "      <td>0.000433</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>399.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>max specificity</td>\n",
       "      <td>0.952242</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>max absolute_mcc</td>\n",
       "      <td>0.138112</td>\n",
       "      <td>0.287951</td>\n",
       "      <td>216.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>max min_per_class_accuracy</td>\n",
       "      <td>0.143558</td>\n",
       "      <td>0.725637</td>\n",
       "      <td>211.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>max mean_per_class_accuracy</td>\n",
       "      <td>0.099263</td>\n",
       "      <td>0.735993</td>\n",
       "      <td>260.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>max tns</td>\n",
       "      <td>0.952242</td>\n",
       "      <td>144833.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>max fns</td>\n",
       "      <td>0.952242</td>\n",
       "      <td>15504.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>max fps</td>\n",
       "      <td>0.000433</td>\n",
       "      <td>144833.000000</td>\n",
       "      <td>399.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>max tps</td>\n",
       "      <td>0.000433</td>\n",
       "      <td>15505.000000</td>\n",
       "      <td>399.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>max tnr</td>\n",
       "      <td>0.952242</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>max fnr</td>\n",
       "      <td>0.952242</td>\n",
       "      <td>0.999936</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>max fpr</td>\n",
       "      <td>0.000433</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>399.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>max tpr</td>\n",
       "      <td>0.000433</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>399.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         metric  threshold          value    idx\n",
       "0                        max f1   0.163276       0.343995  189.0\n",
       "1                        max f2   0.106512       0.507641  252.0\n",
       "2                  max f0point5   0.215648       0.285605  140.0\n",
       "3                  max accuracy   0.952242       0.903304    0.0\n",
       "4                 max precision   0.952242       1.000000    0.0\n",
       "5                    max recall   0.000433       1.000000  399.0\n",
       "6               max specificity   0.952242       1.000000    0.0\n",
       "7              max absolute_mcc   0.138112       0.287951  216.0\n",
       "8    max min_per_class_accuracy   0.143558       0.725637  211.0\n",
       "9   max mean_per_class_accuracy   0.099263       0.735993  260.0\n",
       "10                      max tns   0.952242  144833.000000    0.0\n",
       "11                      max fns   0.952242   15504.000000    0.0\n",
       "12                      max fps   0.000433  144833.000000  399.0\n",
       "13                      max tps   0.000433   15505.000000  399.0\n",
       "14                      max tnr   0.952242       1.000000    0.0\n",
       "15                      max fnr   0.952242       0.999936    0.0\n",
       "16                      max fpr   0.000433       1.000000  399.0\n",
       "17                      max tpr   0.000433       1.000000  399.0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Gains/Lift Table: Avg response rate:  9.67 %, avg score:  9.69 %\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>group</th>\n",
       "      <th>cumulative_data_fraction</th>\n",
       "      <th>lower_threshold</th>\n",
       "      <th>lift</th>\n",
       "      <th>cumulative_lift</th>\n",
       "      <th>response_rate</th>\n",
       "      <th>score</th>\n",
       "      <th>cumulative_response_rate</th>\n",
       "      <th>cumulative_score</th>\n",
       "      <th>capture_rate</th>\n",
       "      <th>cumulative_capture_rate</th>\n",
       "      <th>gain</th>\n",
       "      <th>cumulative_gain</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "      <td>0.010353</td>\n",
       "      <td>0.354732</td>\n",
       "      <td>3.806254</td>\n",
       "      <td>3.806254</td>\n",
       "      <td>0.368072</td>\n",
       "      <td>0.418577</td>\n",
       "      <td>0.368072</td>\n",
       "      <td>0.418577</td>\n",
       "      <td>0.039407</td>\n",
       "      <td>0.039407</td>\n",
       "      <td>280.625441</td>\n",
       "      <td>280.625441</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td></td>\n",
       "      <td>2</td>\n",
       "      <td>0.020045</td>\n",
       "      <td>0.324773</td>\n",
       "      <td>3.393781</td>\n",
       "      <td>3.606820</td>\n",
       "      <td>0.328185</td>\n",
       "      <td>0.338285</td>\n",
       "      <td>0.348787</td>\n",
       "      <td>0.379755</td>\n",
       "      <td>0.032893</td>\n",
       "      <td>0.072299</td>\n",
       "      <td>239.378131</td>\n",
       "      <td>260.681969</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td></td>\n",
       "      <td>3</td>\n",
       "      <td>0.030286</td>\n",
       "      <td>0.304131</td>\n",
       "      <td>3.331557</td>\n",
       "      <td>3.513743</td>\n",
       "      <td>0.322168</td>\n",
       "      <td>0.316421</td>\n",
       "      <td>0.339786</td>\n",
       "      <td>0.358339</td>\n",
       "      <td>0.034118</td>\n",
       "      <td>0.106417</td>\n",
       "      <td>233.155671</td>\n",
       "      <td>251.374271</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td></td>\n",
       "      <td>4</td>\n",
       "      <td>0.040109</td>\n",
       "      <td>0.273867</td>\n",
       "      <td>2.764179</td>\n",
       "      <td>3.330169</td>\n",
       "      <td>0.267302</td>\n",
       "      <td>0.285482</td>\n",
       "      <td>0.322034</td>\n",
       "      <td>0.340496</td>\n",
       "      <td>0.027153</td>\n",
       "      <td>0.133570</td>\n",
       "      <td>176.417942</td>\n",
       "      <td>233.016905</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td></td>\n",
       "      <td>5</td>\n",
       "      <td>0.050200</td>\n",
       "      <td>0.263173</td>\n",
       "      <td>2.543720</td>\n",
       "      <td>3.172078</td>\n",
       "      <td>0.245983</td>\n",
       "      <td>0.267200</td>\n",
       "      <td>0.306746</td>\n",
       "      <td>0.325762</td>\n",
       "      <td>0.025669</td>\n",
       "      <td>0.159239</td>\n",
       "      <td>154.371966</td>\n",
       "      <td>217.207797</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td></td>\n",
       "      <td>6</td>\n",
       "      <td>0.100051</td>\n",
       "      <td>0.239285</td>\n",
       "      <td>2.661271</td>\n",
       "      <td>2.917566</td>\n",
       "      <td>0.257350</td>\n",
       "      <td>0.250949</td>\n",
       "      <td>0.282134</td>\n",
       "      <td>0.288486</td>\n",
       "      <td>0.132667</td>\n",
       "      <td>0.291906</td>\n",
       "      <td>166.127142</td>\n",
       "      <td>191.756627</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td></td>\n",
       "      <td>7</td>\n",
       "      <td>0.150008</td>\n",
       "      <td>0.214789</td>\n",
       "      <td>2.389674</td>\n",
       "      <td>2.741763</td>\n",
       "      <td>0.231086</td>\n",
       "      <td>0.225620</td>\n",
       "      <td>0.265134</td>\n",
       "      <td>0.267550</td>\n",
       "      <td>0.119381</td>\n",
       "      <td>0.411287</td>\n",
       "      <td>138.967365</td>\n",
       "      <td>174.176301</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td></td>\n",
       "      <td>8</td>\n",
       "      <td>0.200121</td>\n",
       "      <td>0.189296</td>\n",
       "      <td>2.168596</td>\n",
       "      <td>2.598235</td>\n",
       "      <td>0.209708</td>\n",
       "      <td>0.202499</td>\n",
       "      <td>0.251254</td>\n",
       "      <td>0.251260</td>\n",
       "      <td>0.108675</td>\n",
       "      <td>0.519961</td>\n",
       "      <td>116.859632</td>\n",
       "      <td>159.823465</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td></td>\n",
       "      <td>9</td>\n",
       "      <td>0.300472</td>\n",
       "      <td>0.148716</td>\n",
       "      <td>1.782209</td>\n",
       "      <td>2.325701</td>\n",
       "      <td>0.172343</td>\n",
       "      <td>0.168564</td>\n",
       "      <td>0.224900</td>\n",
       "      <td>0.223642</td>\n",
       "      <td>0.178846</td>\n",
       "      <td>0.698807</td>\n",
       "      <td>78.220853</td>\n",
       "      <td>132.570086</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td></td>\n",
       "      <td>10</td>\n",
       "      <td>0.400049</td>\n",
       "      <td>0.107141</td>\n",
       "      <td>1.267533</td>\n",
       "      <td>2.062310</td>\n",
       "      <td>0.122573</td>\n",
       "      <td>0.128001</td>\n",
       "      <td>0.199429</td>\n",
       "      <td>0.199836</td>\n",
       "      <td>0.126217</td>\n",
       "      <td>0.825024</td>\n",
       "      <td>26.753334</td>\n",
       "      <td>106.230965</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td></td>\n",
       "      <td>11</td>\n",
       "      <td>0.500056</td>\n",
       "      <td>0.066583</td>\n",
       "      <td>0.825478</td>\n",
       "      <td>1.814953</td>\n",
       "      <td>0.079825</td>\n",
       "      <td>0.086946</td>\n",
       "      <td>0.175509</td>\n",
       "      <td>0.177259</td>\n",
       "      <td>0.082554</td>\n",
       "      <td>0.907578</td>\n",
       "      <td>-17.452163</td>\n",
       "      <td>81.495265</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td></td>\n",
       "      <td>12</td>\n",
       "      <td>0.600032</td>\n",
       "      <td>0.031446</td>\n",
       "      <td>0.496087</td>\n",
       "      <td>1.595206</td>\n",
       "      <td>0.047973</td>\n",
       "      <td>0.047414</td>\n",
       "      <td>0.154260</td>\n",
       "      <td>0.155624</td>\n",
       "      <td>0.049597</td>\n",
       "      <td>0.957175</td>\n",
       "      <td>-50.391339</td>\n",
       "      <td>59.520562</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td></td>\n",
       "      <td>13</td>\n",
       "      <td>0.699996</td>\n",
       "      <td>0.011776</td>\n",
       "      <td>0.241300</td>\n",
       "      <td>1.401859</td>\n",
       "      <td>0.023334</td>\n",
       "      <td>0.020903</td>\n",
       "      <td>0.135563</td>\n",
       "      <td>0.136385</td>\n",
       "      <td>0.024121</td>\n",
       "      <td>0.981296</td>\n",
       "      <td>-75.870020</td>\n",
       "      <td>40.185943</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td></td>\n",
       "      <td>14</td>\n",
       "      <td>0.800010</td>\n",
       "      <td>0.005632</td>\n",
       "      <td>0.094150</td>\n",
       "      <td>1.238375</td>\n",
       "      <td>0.009105</td>\n",
       "      <td>0.008023</td>\n",
       "      <td>0.119753</td>\n",
       "      <td>0.120338</td>\n",
       "      <td>0.009416</td>\n",
       "      <td>0.990713</td>\n",
       "      <td>-90.584975</td>\n",
       "      <td>23.837539</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td></td>\n",
       "      <td>15</td>\n",
       "      <td>0.900042</td>\n",
       "      <td>0.003187</td>\n",
       "      <td>0.055448</td>\n",
       "      <td>1.106903</td>\n",
       "      <td>0.005362</td>\n",
       "      <td>0.004345</td>\n",
       "      <td>0.107040</td>\n",
       "      <td>0.107446</td>\n",
       "      <td>0.005547</td>\n",
       "      <td>0.996259</td>\n",
       "      <td>-94.455200</td>\n",
       "      <td>10.690259</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td></td>\n",
       "      <td>16</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000023</td>\n",
       "      <td>0.037423</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.003619</td>\n",
       "      <td>0.002011</td>\n",
       "      <td>0.096702</td>\n",
       "      <td>0.096907</td>\n",
       "      <td>0.003741</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-96.257684</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      group  cumulative_data_fraction  lower_threshold      lift  \\\n",
       "0         1                  0.010353         0.354732  3.806254   \n",
       "1         2                  0.020045         0.324773  3.393781   \n",
       "2         3                  0.030286         0.304131  3.331557   \n",
       "3         4                  0.040109         0.273867  2.764179   \n",
       "4         5                  0.050200         0.263173  2.543720   \n",
       "5         6                  0.100051         0.239285  2.661271   \n",
       "6         7                  0.150008         0.214789  2.389674   \n",
       "7         8                  0.200121         0.189296  2.168596   \n",
       "8         9                  0.300472         0.148716  1.782209   \n",
       "9        10                  0.400049         0.107141  1.267533   \n",
       "10       11                  0.500056         0.066583  0.825478   \n",
       "11       12                  0.600032         0.031446  0.496087   \n",
       "12       13                  0.699996         0.011776  0.241300   \n",
       "13       14                  0.800010         0.005632  0.094150   \n",
       "14       15                  0.900042         0.003187  0.055448   \n",
       "15       16                  1.000000         0.000023  0.037423   \n",
       "\n",
       "    cumulative_lift  response_rate     score  cumulative_response_rate  \\\n",
       "0          3.806254       0.368072  0.418577                  0.368072   \n",
       "1          3.606820       0.328185  0.338285                  0.348787   \n",
       "2          3.513743       0.322168  0.316421                  0.339786   \n",
       "3          3.330169       0.267302  0.285482                  0.322034   \n",
       "4          3.172078       0.245983  0.267200                  0.306746   \n",
       "5          2.917566       0.257350  0.250949                  0.282134   \n",
       "6          2.741763       0.231086  0.225620                  0.265134   \n",
       "7          2.598235       0.209708  0.202499                  0.251254   \n",
       "8          2.325701       0.172343  0.168564                  0.224900   \n",
       "9          2.062310       0.122573  0.128001                  0.199429   \n",
       "10         1.814953       0.079825  0.086946                  0.175509   \n",
       "11         1.595206       0.047973  0.047414                  0.154260   \n",
       "12         1.401859       0.023334  0.020903                  0.135563   \n",
       "13         1.238375       0.009105  0.008023                  0.119753   \n",
       "14         1.106903       0.005362  0.004345                  0.107040   \n",
       "15         1.000000       0.003619  0.002011                  0.096702   \n",
       "\n",
       "    cumulative_score  capture_rate  cumulative_capture_rate        gain  \\\n",
       "0           0.418577      0.039407                 0.039407  280.625441   \n",
       "1           0.379755      0.032893                 0.072299  239.378131   \n",
       "2           0.358339      0.034118                 0.106417  233.155671   \n",
       "3           0.340496      0.027153                 0.133570  176.417942   \n",
       "4           0.325762      0.025669                 0.159239  154.371966   \n",
       "5           0.288486      0.132667                 0.291906  166.127142   \n",
       "6           0.267550      0.119381                 0.411287  138.967365   \n",
       "7           0.251260      0.108675                 0.519961  116.859632   \n",
       "8           0.223642      0.178846                 0.698807   78.220853   \n",
       "9           0.199836      0.126217                 0.825024   26.753334   \n",
       "10          0.177259      0.082554                 0.907578  -17.452163   \n",
       "11          0.155624      0.049597                 0.957175  -50.391339   \n",
       "12          0.136385      0.024121                 0.981296  -75.870020   \n",
       "13          0.120338      0.009416                 0.990713  -90.584975   \n",
       "14          0.107446      0.005547                 0.996259  -94.455200   \n",
       "15          0.096907      0.003741                 1.000000  -96.257684   \n",
       "\n",
       "    cumulative_gain  \n",
       "0        280.625441  \n",
       "1        260.681969  \n",
       "2        251.374271  \n",
       "3        233.016905  \n",
       "4        217.207797  \n",
       "5        191.756627  \n",
       "6        174.176301  \n",
       "7        159.823465  \n",
       "8        132.570086  \n",
       "9        106.230965  \n",
       "10        81.495265  \n",
       "11        59.520562  \n",
       "12        40.185943  \n",
       "13        23.837539  \n",
       "14        10.690259  \n",
       "15         0.000000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Cross-Validation Metrics Summary: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>sd</th>\n",
       "      <th>cv_1_valid</th>\n",
       "      <th>cv_2_valid</th>\n",
       "      <th>cv_3_valid</th>\n",
       "      <th>cv_4_valid</th>\n",
       "      <th>cv_5_valid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>accuracy</td>\n",
       "      <td>0.77324414</td>\n",
       "      <td>0.019544147</td>\n",
       "      <td>0.76724625</td>\n",
       "      <td>0.8054997</td>\n",
       "      <td>0.75335455</td>\n",
       "      <td>0.76583946</td>\n",
       "      <td>0.77428067</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>auc</td>\n",
       "      <td>0.79898</td>\n",
       "      <td>0.0034089838</td>\n",
       "      <td>0.7950255</td>\n",
       "      <td>0.8023207</td>\n",
       "      <td>0.7981366</td>\n",
       "      <td>0.8027014</td>\n",
       "      <td>0.7967159</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>err</td>\n",
       "      <td>0.22675589</td>\n",
       "      <td>0.019544147</td>\n",
       "      <td>0.23275377</td>\n",
       "      <td>0.19450031</td>\n",
       "      <td>0.24664545</td>\n",
       "      <td>0.23416054</td>\n",
       "      <td>0.22571935</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>err_count</td>\n",
       "      <td>7273.2</td>\n",
       "      <td>648.7185</td>\n",
       "      <td>7443.0</td>\n",
       "      <td>6189.0</td>\n",
       "      <td>7904.0</td>\n",
       "      <td>7558.0</td>\n",
       "      <td>7272.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>f0point5</td>\n",
       "      <td>0.27366853</td>\n",
       "      <td>0.013980155</td>\n",
       "      <td>0.26927122</td>\n",
       "      <td>0.29628605</td>\n",
       "      <td>0.26027754</td>\n",
       "      <td>0.26578072</td>\n",
       "      <td>0.27672705</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>f1</td>\n",
       "      <td>0.3450911</td>\n",
       "      <td>0.0075461566</td>\n",
       "      <td>0.3432454</td>\n",
       "      <td>0.35618433</td>\n",
       "      <td>0.33680147</td>\n",
       "      <td>0.34060374</td>\n",
       "      <td>0.34862056</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>f2</td>\n",
       "      <td>0.46836054</td>\n",
       "      <td>0.012446837</td>\n",
       "      <td>0.47325903</td>\n",
       "      <td>0.4464379</td>\n",
       "      <td>0.47706205</td>\n",
       "      <td>0.47406256</td>\n",
       "      <td>0.47098118</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>lift_top_group</td>\n",
       "      <td>3.808692</td>\n",
       "      <td>0.2287151</td>\n",
       "      <td>3.6756728</td>\n",
       "      <td>3.5257099</td>\n",
       "      <td>4.10965</td>\n",
       "      <td>3.9507008</td>\n",
       "      <td>3.7817264</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>logloss</td>\n",
       "      <td>0.26392356</td>\n",
       "      <td>0.0048803934</td>\n",
       "      <td>0.26530334</td>\n",
       "      <td>0.26852083</td>\n",
       "      <td>0.26026046</td>\n",
       "      <td>0.25746855</td>\n",
       "      <td>0.26806465</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>max_per_class_error</td>\n",
       "      <td>0.3825464</td>\n",
       "      <td>0.047772843</td>\n",
       "      <td>0.36686197</td>\n",
       "      <td>0.4628177</td>\n",
       "      <td>0.3395854</td>\n",
       "      <td>0.3583169</td>\n",
       "      <td>0.38515007</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>mcc</td>\n",
       "      <td>0.27895746</td>\n",
       "      <td>0.001112472</td>\n",
       "      <td>0.27914262</td>\n",
       "      <td>0.27859268</td>\n",
       "      <td>0.27724242</td>\n",
       "      <td>0.2797517</td>\n",
       "      <td>0.28005788</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>mean_per_class_accuracy</td>\n",
       "      <td>0.70376307</td>\n",
       "      <td>0.010299629</td>\n",
       "      <td>0.70731837</td>\n",
       "      <td>0.6862736</td>\n",
       "      <td>0.71175313</td>\n",
       "      <td>0.7102207</td>\n",
       "      <td>0.7032497</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>mean_per_class_error</td>\n",
       "      <td>0.2962369</td>\n",
       "      <td>0.010299629</td>\n",
       "      <td>0.29268166</td>\n",
       "      <td>0.31372645</td>\n",
       "      <td>0.28824687</td>\n",
       "      <td>0.28977928</td>\n",
       "      <td>0.2967503</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>mse</td>\n",
       "      <td>0.078284055</td>\n",
       "      <td>0.001608798</td>\n",
       "      <td>0.07816433</td>\n",
       "      <td>0.08040479</td>\n",
       "      <td>0.07697405</td>\n",
       "      <td>0.0765446</td>\n",
       "      <td>0.079332516</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>precision</td>\n",
       "      <td>0.24060194</td>\n",
       "      <td>0.015726406</td>\n",
       "      <td>0.23544365</td>\n",
       "      <td>0.26641768</td>\n",
       "      <td>0.22603896</td>\n",
       "      <td>0.23182897</td>\n",
       "      <td>0.24328041</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>r2</td>\n",
       "      <td>0.10376357</td>\n",
       "      <td>0.0028654204</td>\n",
       "      <td>0.09987673</td>\n",
       "      <td>0.10785931</td>\n",
       "      <td>0.103276625</td>\n",
       "      <td>0.10331774</td>\n",
       "      <td>0.10448746</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>recall</td>\n",
       "      <td>0.6174536</td>\n",
       "      <td>0.047772843</td>\n",
       "      <td>0.633138</td>\n",
       "      <td>0.53718233</td>\n",
       "      <td>0.66041464</td>\n",
       "      <td>0.6416831</td>\n",
       "      <td>0.6148499</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>rmse</td>\n",
       "      <td>0.27978107</td>\n",
       "      <td>0.0028723667</td>\n",
       "      <td>0.27957883</td>\n",
       "      <td>0.2835574</td>\n",
       "      <td>0.27744198</td>\n",
       "      <td>0.27666694</td>\n",
       "      <td>0.2816603</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>specificity</td>\n",
       "      <td>0.79007256</td>\n",
       "      <td>0.027310094</td>\n",
       "      <td>0.7814987</td>\n",
       "      <td>0.8353648</td>\n",
       "      <td>0.7630917</td>\n",
       "      <td>0.77875835</td>\n",
       "      <td>0.79164946</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    mean            sd  cv_1_valid  \\\n",
       "0                  accuracy   0.77324414   0.019544147  0.76724625   \n",
       "1                       auc      0.79898  0.0034089838   0.7950255   \n",
       "2                       err   0.22675589   0.019544147  0.23275377   \n",
       "3                 err_count       7273.2      648.7185      7443.0   \n",
       "4                  f0point5   0.27366853   0.013980155  0.26927122   \n",
       "5                        f1    0.3450911  0.0075461566   0.3432454   \n",
       "6                        f2   0.46836054   0.012446837  0.47325903   \n",
       "7            lift_top_group     3.808692     0.2287151   3.6756728   \n",
       "8                   logloss   0.26392356  0.0048803934  0.26530334   \n",
       "9       max_per_class_error    0.3825464   0.047772843  0.36686197   \n",
       "10                      mcc   0.27895746   0.001112472  0.27914262   \n",
       "11  mean_per_class_accuracy   0.70376307   0.010299629  0.70731837   \n",
       "12     mean_per_class_error    0.2962369   0.010299629  0.29268166   \n",
       "13                      mse  0.078284055   0.001608798  0.07816433   \n",
       "14                precision   0.24060194   0.015726406  0.23544365   \n",
       "15                       r2   0.10376357  0.0028654204  0.09987673   \n",
       "16                   recall    0.6174536   0.047772843    0.633138   \n",
       "17                     rmse   0.27978107  0.0028723667  0.27957883   \n",
       "18              specificity   0.79007256   0.027310094   0.7814987   \n",
       "\n",
       "    cv_2_valid   cv_3_valid  cv_4_valid   cv_5_valid  \n",
       "0    0.8054997   0.75335455  0.76583946   0.77428067  \n",
       "1    0.8023207    0.7981366   0.8027014    0.7967159  \n",
       "2   0.19450031   0.24664545  0.23416054   0.22571935  \n",
       "3       6189.0       7904.0      7558.0       7272.0  \n",
       "4   0.29628605   0.26027754  0.26578072   0.27672705  \n",
       "5   0.35618433   0.33680147  0.34060374   0.34862056  \n",
       "6    0.4464379   0.47706205  0.47406256   0.47098118  \n",
       "7    3.5257099      4.10965   3.9507008    3.7817264  \n",
       "8   0.26852083   0.26026046  0.25746855   0.26806465  \n",
       "9    0.4628177    0.3395854   0.3583169   0.38515007  \n",
       "10  0.27859268   0.27724242   0.2797517   0.28005788  \n",
       "11   0.6862736   0.71175313   0.7102207    0.7032497  \n",
       "12  0.31372645   0.28824687  0.28977928    0.2967503  \n",
       "13  0.08040479   0.07697405   0.0765446  0.079332516  \n",
       "14  0.26641768   0.22603896  0.23182897   0.24328041  \n",
       "15  0.10785931  0.103276625  0.10331774   0.10448746  \n",
       "16  0.53718233   0.66041464   0.6416831    0.6148499  \n",
       "17   0.2835574   0.27744198  0.27666694    0.2816603  \n",
       "18   0.8353648    0.7630917  0.77875835   0.79164946  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Scoring History: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>timestamp</th>\n",
       "      <th>duration</th>\n",
       "      <th>number_of_trees</th>\n",
       "      <th>training_rmse</th>\n",
       "      <th>training_logloss</th>\n",
       "      <th>training_auc</th>\n",
       "      <th>training_pr_auc</th>\n",
       "      <th>training_lift</th>\n",
       "      <th>training_classification_error</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:21:58</td>\n",
       "      <td>23.359 sec</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.693147</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.903298</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:21:59</td>\n",
       "      <td>23.780 sec</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.405190</td>\n",
       "      <td>0.517787</td>\n",
       "      <td>0.785280</td>\n",
       "      <td>0.246994</td>\n",
       "      <td>3.532712</td>\n",
       "      <td>0.237024</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:21:59</td>\n",
       "      <td>23.837 sec</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.350527</td>\n",
       "      <td>0.423514</td>\n",
       "      <td>0.787615</td>\n",
       "      <td>0.249774</td>\n",
       "      <td>4.024719</td>\n",
       "      <td>0.237186</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:21:59</td>\n",
       "      <td>23.884 sec</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.319052</td>\n",
       "      <td>0.366969</td>\n",
       "      <td>0.788270</td>\n",
       "      <td>0.249770</td>\n",
       "      <td>4.032035</td>\n",
       "      <td>0.224102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:21:59</td>\n",
       "      <td>23.933 sec</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.301347</td>\n",
       "      <td>0.331397</td>\n",
       "      <td>0.789771</td>\n",
       "      <td>0.250441</td>\n",
       "      <td>4.029504</td>\n",
       "      <td>0.224095</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:21:59</td>\n",
       "      <td>23.989 sec</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.291633</td>\n",
       "      <td>0.308583</td>\n",
       "      <td>0.791773</td>\n",
       "      <td>0.251509</td>\n",
       "      <td>4.067138</td>\n",
       "      <td>0.227557</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:21:59</td>\n",
       "      <td>24.082 sec</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.286340</td>\n",
       "      <td>0.293630</td>\n",
       "      <td>0.792451</td>\n",
       "      <td>0.251672</td>\n",
       "      <td>4.064393</td>\n",
       "      <td>0.218769</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:21:59</td>\n",
       "      <td>24.137 sec</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.283464</td>\n",
       "      <td>0.283752</td>\n",
       "      <td>0.793673</td>\n",
       "      <td>0.252398</td>\n",
       "      <td>4.069869</td>\n",
       "      <td>0.227307</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:21:59</td>\n",
       "      <td>24.212 sec</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.281931</td>\n",
       "      <td>0.277220</td>\n",
       "      <td>0.794130</td>\n",
       "      <td>0.252655</td>\n",
       "      <td>4.086227</td>\n",
       "      <td>0.218308</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:21:59</td>\n",
       "      <td>24.290 sec</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.281083</td>\n",
       "      <td>0.272918</td>\n",
       "      <td>0.794707</td>\n",
       "      <td>0.253553</td>\n",
       "      <td>4.071328</td>\n",
       "      <td>0.226303</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:21:59</td>\n",
       "      <td>24.362 sec</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.280642</td>\n",
       "      <td>0.270039</td>\n",
       "      <td>0.794736</td>\n",
       "      <td>0.253742</td>\n",
       "      <td>4.076169</td>\n",
       "      <td>0.231698</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:21:59</td>\n",
       "      <td>24.436 sec</td>\n",
       "      <td>11.0</td>\n",
       "      <td>0.280415</td>\n",
       "      <td>0.268234</td>\n",
       "      <td>0.794602</td>\n",
       "      <td>0.255840</td>\n",
       "      <td>4.088889</td>\n",
       "      <td>0.224588</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:21:59</td>\n",
       "      <td>24.511 sec</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.280268</td>\n",
       "      <td>0.266927</td>\n",
       "      <td>0.794947</td>\n",
       "      <td>0.256068</td>\n",
       "      <td>4.101077</td>\n",
       "      <td>0.220422</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:21:59</td>\n",
       "      <td>24.588 sec</td>\n",
       "      <td>13.0</td>\n",
       "      <td>0.280166</td>\n",
       "      <td>0.266017</td>\n",
       "      <td>0.795440</td>\n",
       "      <td>0.256359</td>\n",
       "      <td>4.092700</td>\n",
       "      <td>0.220135</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:21:59</td>\n",
       "      <td>24.665 sec</td>\n",
       "      <td>14.0</td>\n",
       "      <td>0.280086</td>\n",
       "      <td>0.265479</td>\n",
       "      <td>0.795846</td>\n",
       "      <td>0.256787</td>\n",
       "      <td>4.090298</td>\n",
       "      <td>0.220066</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:21:59</td>\n",
       "      <td>24.749 sec</td>\n",
       "      <td>15.0</td>\n",
       "      <td>0.280034</td>\n",
       "      <td>0.265065</td>\n",
       "      <td>0.796107</td>\n",
       "      <td>0.257183</td>\n",
       "      <td>4.102134</td>\n",
       "      <td>0.231916</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:22:00</td>\n",
       "      <td>24.841 sec</td>\n",
       "      <td>16.0</td>\n",
       "      <td>0.280027</td>\n",
       "      <td>0.264886</td>\n",
       "      <td>0.796078</td>\n",
       "      <td>0.257197</td>\n",
       "      <td>4.094861</td>\n",
       "      <td>0.216031</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:22:00</td>\n",
       "      <td>24.924 sec</td>\n",
       "      <td>17.0</td>\n",
       "      <td>0.280000</td>\n",
       "      <td>0.264732</td>\n",
       "      <td>0.796132</td>\n",
       "      <td>0.257623</td>\n",
       "      <td>4.105826</td>\n",
       "      <td>0.216131</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:22:00</td>\n",
       "      <td>25.011 sec</td>\n",
       "      <td>18.0</td>\n",
       "      <td>0.279884</td>\n",
       "      <td>0.264171</td>\n",
       "      <td>0.797136</td>\n",
       "      <td>0.258124</td>\n",
       "      <td>4.079334</td>\n",
       "      <td>0.215925</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:22:00</td>\n",
       "      <td>25.119 sec</td>\n",
       "      <td>19.0</td>\n",
       "      <td>0.279798</td>\n",
       "      <td>0.263740</td>\n",
       "      <td>0.797849</td>\n",
       "      <td>0.258385</td>\n",
       "      <td>4.065712</td>\n",
       "      <td>0.216162</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                timestamp    duration  number_of_trees  training_rmse  \\\n",
       "0     2019-12-09 13:21:58  23.359 sec              0.0       0.500000   \n",
       "1     2019-12-09 13:21:59  23.780 sec              1.0       0.405190   \n",
       "2     2019-12-09 13:21:59  23.837 sec              2.0       0.350527   \n",
       "3     2019-12-09 13:21:59  23.884 sec              3.0       0.319052   \n",
       "4     2019-12-09 13:21:59  23.933 sec              4.0       0.301347   \n",
       "5     2019-12-09 13:21:59  23.989 sec              5.0       0.291633   \n",
       "6     2019-12-09 13:21:59  24.082 sec              6.0       0.286340   \n",
       "7     2019-12-09 13:21:59  24.137 sec              7.0       0.283464   \n",
       "8     2019-12-09 13:21:59  24.212 sec              8.0       0.281931   \n",
       "9     2019-12-09 13:21:59  24.290 sec              9.0       0.281083   \n",
       "10    2019-12-09 13:21:59  24.362 sec             10.0       0.280642   \n",
       "11    2019-12-09 13:21:59  24.436 sec             11.0       0.280415   \n",
       "12    2019-12-09 13:21:59  24.511 sec             12.0       0.280268   \n",
       "13    2019-12-09 13:21:59  24.588 sec             13.0       0.280166   \n",
       "14    2019-12-09 13:21:59  24.665 sec             14.0       0.280086   \n",
       "15    2019-12-09 13:21:59  24.749 sec             15.0       0.280034   \n",
       "16    2019-12-09 13:22:00  24.841 sec             16.0       0.280027   \n",
       "17    2019-12-09 13:22:00  24.924 sec             17.0       0.280000   \n",
       "18    2019-12-09 13:22:00  25.011 sec             18.0       0.279884   \n",
       "19    2019-12-09 13:22:00  25.119 sec             19.0       0.279798   \n",
       "\n",
       "    training_logloss  training_auc  training_pr_auc  training_lift  \\\n",
       "0           0.693147      0.500000         0.000000       1.000000   \n",
       "1           0.517787      0.785280         0.246994       3.532712   \n",
       "2           0.423514      0.787615         0.249774       4.024719   \n",
       "3           0.366969      0.788270         0.249770       4.032035   \n",
       "4           0.331397      0.789771         0.250441       4.029504   \n",
       "5           0.308583      0.791773         0.251509       4.067138   \n",
       "6           0.293630      0.792451         0.251672       4.064393   \n",
       "7           0.283752      0.793673         0.252398       4.069869   \n",
       "8           0.277220      0.794130         0.252655       4.086227   \n",
       "9           0.272918      0.794707         0.253553       4.071328   \n",
       "10          0.270039      0.794736         0.253742       4.076169   \n",
       "11          0.268234      0.794602         0.255840       4.088889   \n",
       "12          0.266927      0.794947         0.256068       4.101077   \n",
       "13          0.266017      0.795440         0.256359       4.092700   \n",
       "14          0.265479      0.795846         0.256787       4.090298   \n",
       "15          0.265065      0.796107         0.257183       4.102134   \n",
       "16          0.264886      0.796078         0.257197       4.094861   \n",
       "17          0.264732      0.796132         0.257623       4.105826   \n",
       "18          0.264171      0.797136         0.258124       4.079334   \n",
       "19          0.263740      0.797849         0.258385       4.065712   \n",
       "\n",
       "    training_classification_error  \n",
       "0                        0.903298  \n",
       "1                        0.237024  \n",
       "2                        0.237186  \n",
       "3                        0.224102  \n",
       "4                        0.224095  \n",
       "5                        0.227557  \n",
       "6                        0.218769  \n",
       "7                        0.227307  \n",
       "8                        0.218308  \n",
       "9                        0.226303  \n",
       "10                       0.231698  \n",
       "11                       0.224588  \n",
       "12                       0.220422  \n",
       "13                       0.220135  \n",
       "14                       0.220066  \n",
       "15                       0.231916  \n",
       "16                       0.216031  \n",
       "17                       0.216131  \n",
       "18                       0.215925  \n",
       "19                       0.216162  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "See the whole table with table.as_data_frame()\n",
      "\n",
      "Variable Importances: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>variable</th>\n",
       "      <th>relative_importance</th>\n",
       "      <th>scaled_importance</th>\n",
       "      <th>percentage</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>loan_to_value_ratio_std</td>\n",
       "      <td>17366.339844</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.548268</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>property_value_std</td>\n",
       "      <td>4905.364258</td>\n",
       "      <td>0.282464</td>\n",
       "      <td>0.154866</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>loan_amount_std</td>\n",
       "      <td>3695.568604</td>\n",
       "      <td>0.212801</td>\n",
       "      <td>0.116672</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>debt_to_income_ratio_std</td>\n",
       "      <td>2890.878418</td>\n",
       "      <td>0.166464</td>\n",
       "      <td>0.091267</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>intro_rate_period_std</td>\n",
       "      <td>1570.096191</td>\n",
       "      <td>0.090410</td>\n",
       "      <td>0.049569</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>no_intro_rate_period_std</td>\n",
       "      <td>926.157043</td>\n",
       "      <td>0.053331</td>\n",
       "      <td>0.029239</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>income_std</td>\n",
       "      <td>215.108414</td>\n",
       "      <td>0.012387</td>\n",
       "      <td>0.006791</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>conforming</td>\n",
       "      <td>89.246017</td>\n",
       "      <td>0.005139</td>\n",
       "      <td>0.002818</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>debt_to_income_ratio_missing.False</td>\n",
       "      <td>14.767391</td>\n",
       "      <td>0.000850</td>\n",
       "      <td>0.000466</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>term_360</td>\n",
       "      <td>1.370327</td>\n",
       "      <td>0.000079</td>\n",
       "      <td>0.000043</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             variable  relative_importance  scaled_importance  \\\n",
       "0             loan_to_value_ratio_std         17366.339844           1.000000   \n",
       "1                  property_value_std          4905.364258           0.282464   \n",
       "2                     loan_amount_std          3695.568604           0.212801   \n",
       "3            debt_to_income_ratio_std          2890.878418           0.166464   \n",
       "4               intro_rate_period_std          1570.096191           0.090410   \n",
       "5            no_intro_rate_period_std           926.157043           0.053331   \n",
       "6                          income_std           215.108414           0.012387   \n",
       "7                          conforming            89.246017           0.005139   \n",
       "8  debt_to_income_ratio_missing.False            14.767391           0.000850   \n",
       "9                            term_360             1.370327           0.000079   \n",
       "\n",
       "   percentage  \n",
       "0    0.548268  \n",
       "1    0.154866  \n",
       "2    0.116672  \n",
       "3    0.091267  \n",
       "4    0.049569  \n",
       "5    0.029239  \n",
       "6    0.006791  \n",
       "7    0.002818  \n",
       "8    0.000466  \n",
       "9    0.000043  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": []
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mxgb_default"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8031621930708982\n"
     ]
    }
   ],
   "source": [
    "# Get the AUC on the test set for MXGB\n",
    "perf_mxgb_default = mxgb_default.model_performance(test)\n",
    "print(perf_mxgb_default.auc())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.798732101317958\n"
     ]
    }
   ],
   "source": [
    "# Get the AUC on 5 fold CV for the default MXGB\n",
    "perf_cv_mxgb_default = mxgb_default.model_performance(xval=True)\n",
    "print(perf_cv_mxgb_default.auc())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Build Default XGBoost with NO Monotonic Constraints"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "xgboost Model Build progress: |███████████████████████████████████████████| 100%\n"
     ]
    }
   ],
   "source": [
    "xgb_default = H2OXGBoostEstimator(\n",
    "               seed=SEED)\n",
    "xgb_default.train(x=X,\n",
    "              y=Y,\n",
    "              training_frame=train,\n",
    "              fold_column=fold_column)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model Summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model Details\n",
      "=============\n",
      "H2OXGBoostEstimator :  XGBoost\n",
      "Model Key:  XGBoost_model_python_1575925687897_1298\n",
      "\n",
      "\n",
      "Model Summary: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>number_of_trees</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td></td>\n",
       "      <td>50.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     number_of_trees\n",
       "0               50.0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "ModelMetricsBinomial: xgboost\n",
      "** Reported on train data. **\n",
      "\n",
      "MSE: 0.07346298335980485\n",
      "RMSE: 0.2710405566696705\n",
      "LogLoss: 0.2421490326043258\n",
      "Mean Per-Class Error: 0.22331836139590344\n",
      "AUC: 0.8436379645315262\n",
      "pr_auc: 0.34572862742360894\n",
      "Gini: 0.6872759290630523\n",
      "\n",
      "Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.18837254450994756: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>Error</th>\n",
       "      <th>Rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>119013.0</td>\n",
       "      <td>25820.0</td>\n",
       "      <td>0.1783</td>\n",
       "      <td>(25820.0/144833.0)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>5469.0</td>\n",
       "      <td>10036.0</td>\n",
       "      <td>0.3527</td>\n",
       "      <td>(5469.0/15505.0)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Total</td>\n",
       "      <td>124482.0</td>\n",
       "      <td>35856.0</td>\n",
       "      <td>0.1951</td>\n",
       "      <td>(31289.0/160338.0)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 0        1   Error                 Rate\n",
       "0      0  119013.0  25820.0  0.1783   (25820.0/144833.0)\n",
       "1      1    5469.0  10036.0  0.3527     (5469.0/15505.0)\n",
       "2  Total  124482.0  35856.0  0.1951   (31289.0/160338.0)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Maximum Metrics: Maximum metrics at their respective thresholds\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>metric</th>\n",
       "      <th>threshold</th>\n",
       "      <th>value</th>\n",
       "      <th>idx</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>max f1</td>\n",
       "      <td>0.188373</td>\n",
       "      <td>0.390802</td>\n",
       "      <td>207.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>max f2</td>\n",
       "      <td>0.116665</td>\n",
       "      <td>0.555551</td>\n",
       "      <td>266.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>max f0point5</td>\n",
       "      <td>0.286709</td>\n",
       "      <td>0.347332</td>\n",
       "      <td>129.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>max accuracy</td>\n",
       "      <td>0.407835</td>\n",
       "      <td>0.905805</td>\n",
       "      <td>71.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>max precision</td>\n",
       "      <td>0.976949</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>max recall</td>\n",
       "      <td>0.002968</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>392.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>max specificity</td>\n",
       "      <td>0.976949</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>max absolute_mcc</td>\n",
       "      <td>0.146853</td>\n",
       "      <td>0.342875</td>\n",
       "      <td>241.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>max min_per_class_accuracy</td>\n",
       "      <td>0.154572</td>\n",
       "      <td>0.760658</td>\n",
       "      <td>235.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>max mean_per_class_accuracy</td>\n",
       "      <td>0.100494</td>\n",
       "      <td>0.776682</td>\n",
       "      <td>280.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>max tns</td>\n",
       "      <td>0.976949</td>\n",
       "      <td>144833.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>max fns</td>\n",
       "      <td>0.976949</td>\n",
       "      <td>15504.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>max fps</td>\n",
       "      <td>0.000418</td>\n",
       "      <td>144833.000000</td>\n",
       "      <td>399.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>max tps</td>\n",
       "      <td>0.002968</td>\n",
       "      <td>15505.000000</td>\n",
       "      <td>392.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>max tnr</td>\n",
       "      <td>0.976949</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>max fnr</td>\n",
       "      <td>0.976949</td>\n",
       "      <td>0.999936</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>max fpr</td>\n",
       "      <td>0.000418</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>399.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>max tpr</td>\n",
       "      <td>0.002968</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>392.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         metric  threshold          value    idx\n",
       "0                        max f1   0.188373       0.390802  207.0\n",
       "1                        max f2   0.116665       0.555551  266.0\n",
       "2                  max f0point5   0.286709       0.347332  129.0\n",
       "3                  max accuracy   0.407835       0.905805   71.0\n",
       "4                 max precision   0.976949       1.000000    0.0\n",
       "5                    max recall   0.002968       1.000000  392.0\n",
       "6               max specificity   0.976949       1.000000    0.0\n",
       "7              max absolute_mcc   0.146853       0.342875  241.0\n",
       "8    max min_per_class_accuracy   0.154572       0.760658  235.0\n",
       "9   max mean_per_class_accuracy   0.100494       0.776682  280.0\n",
       "10                      max tns   0.976949  144833.000000    0.0\n",
       "11                      max fns   0.976949   15504.000000    0.0\n",
       "12                      max fps   0.000418  144833.000000  399.0\n",
       "13                      max tps   0.002968   15505.000000  392.0\n",
       "14                      max tnr   0.976949       1.000000    0.0\n",
       "15                      max fnr   0.976949       0.999936    0.0\n",
       "16                      max fpr   0.000418       1.000000  399.0\n",
       "17                      max tpr   0.002968       1.000000  392.0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Gains/Lift Table: Avg response rate:  9.67 %, avg score:  9.67 %\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>group</th>\n",
       "      <th>cumulative_data_fraction</th>\n",
       "      <th>lower_threshold</th>\n",
       "      <th>lift</th>\n",
       "      <th>cumulative_lift</th>\n",
       "      <th>response_rate</th>\n",
       "      <th>score</th>\n",
       "      <th>cumulative_response_rate</th>\n",
       "      <th>cumulative_score</th>\n",
       "      <th>capture_rate</th>\n",
       "      <th>cumulative_capture_rate</th>\n",
       "      <th>gain</th>\n",
       "      <th>cumulative_gain</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "      <td>0.010029</td>\n",
       "      <td>0.398797</td>\n",
       "      <td>6.283089</td>\n",
       "      <td>6.283089</td>\n",
       "      <td>0.607587</td>\n",
       "      <td>0.504357</td>\n",
       "      <td>0.607587</td>\n",
       "      <td>0.504357</td>\n",
       "      <td>0.063012</td>\n",
       "      <td>0.063012</td>\n",
       "      <td>528.308899</td>\n",
       "      <td>528.308899</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td></td>\n",
       "      <td>2</td>\n",
       "      <td>0.020101</td>\n",
       "      <td>0.355230</td>\n",
       "      <td>4.354127</td>\n",
       "      <td>5.316513</td>\n",
       "      <td>0.421053</td>\n",
       "      <td>0.375510</td>\n",
       "      <td>0.514117</td>\n",
       "      <td>0.439794</td>\n",
       "      <td>0.043857</td>\n",
       "      <td>0.106869</td>\n",
       "      <td>335.412685</td>\n",
       "      <td>431.651317</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td></td>\n",
       "      <td>3</td>\n",
       "      <td>0.030030</td>\n",
       "      <td>0.332043</td>\n",
       "      <td>3.585591</td>\n",
       "      <td>4.744212</td>\n",
       "      <td>0.346734</td>\n",
       "      <td>0.344439</td>\n",
       "      <td>0.458775</td>\n",
       "      <td>0.408267</td>\n",
       "      <td>0.035601</td>\n",
       "      <td>0.142470</td>\n",
       "      <td>258.559064</td>\n",
       "      <td>374.421231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td></td>\n",
       "      <td>4</td>\n",
       "      <td>0.040015</td>\n",
       "      <td>0.312552</td>\n",
       "      <td>3.352283</td>\n",
       "      <td>4.396881</td>\n",
       "      <td>0.324172</td>\n",
       "      <td>0.321365</td>\n",
       "      <td>0.425187</td>\n",
       "      <td>0.386582</td>\n",
       "      <td>0.033473</td>\n",
       "      <td>0.175943</td>\n",
       "      <td>235.228333</td>\n",
       "      <td>339.688090</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td></td>\n",
       "      <td>5</td>\n",
       "      <td>0.050069</td>\n",
       "      <td>0.298603</td>\n",
       "      <td>3.406388</td>\n",
       "      <td>4.197993</td>\n",
       "      <td>0.329404</td>\n",
       "      <td>0.305312</td>\n",
       "      <td>0.405954</td>\n",
       "      <td>0.370263</td>\n",
       "      <td>0.034247</td>\n",
       "      <td>0.210190</td>\n",
       "      <td>240.638848</td>\n",
       "      <td>319.799279</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td></td>\n",
       "      <td>6</td>\n",
       "      <td>0.100001</td>\n",
       "      <td>0.260014</td>\n",
       "      <td>2.839075</td>\n",
       "      <td>3.519466</td>\n",
       "      <td>0.274544</td>\n",
       "      <td>0.277976</td>\n",
       "      <td>0.340339</td>\n",
       "      <td>0.324183</td>\n",
       "      <td>0.141761</td>\n",
       "      <td>0.351951</td>\n",
       "      <td>183.907453</td>\n",
       "      <td>251.946593</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td></td>\n",
       "      <td>7</td>\n",
       "      <td>0.150083</td>\n",
       "      <td>0.229153</td>\n",
       "      <td>2.601360</td>\n",
       "      <td>3.213100</td>\n",
       "      <td>0.251557</td>\n",
       "      <td>0.243968</td>\n",
       "      <td>0.310713</td>\n",
       "      <td>0.297416</td>\n",
       "      <td>0.130281</td>\n",
       "      <td>0.482232</td>\n",
       "      <td>160.136035</td>\n",
       "      <td>221.310008</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td></td>\n",
       "      <td>8</td>\n",
       "      <td>0.200021</td>\n",
       "      <td>0.201311</td>\n",
       "      <td>2.332451</td>\n",
       "      <td>2.993233</td>\n",
       "      <td>0.225553</td>\n",
       "      <td>0.215210</td>\n",
       "      <td>0.289452</td>\n",
       "      <td>0.276892</td>\n",
       "      <td>0.116479</td>\n",
       "      <td>0.598710</td>\n",
       "      <td>133.245143</td>\n",
       "      <td>199.323311</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td></td>\n",
       "      <td>9</td>\n",
       "      <td>0.300047</td>\n",
       "      <td>0.147321</td>\n",
       "      <td>1.807975</td>\n",
       "      <td>2.598106</td>\n",
       "      <td>0.174835</td>\n",
       "      <td>0.174280</td>\n",
       "      <td>0.251242</td>\n",
       "      <td>0.242684</td>\n",
       "      <td>0.180845</td>\n",
       "      <td>0.779555</td>\n",
       "      <td>80.797529</td>\n",
       "      <td>159.810611</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td></td>\n",
       "      <td>10</td>\n",
       "      <td>0.400005</td>\n",
       "      <td>0.090062</td>\n",
       "      <td>1.179475</td>\n",
       "      <td>2.243603</td>\n",
       "      <td>0.114058</td>\n",
       "      <td>0.118626</td>\n",
       "      <td>0.216961</td>\n",
       "      <td>0.211683</td>\n",
       "      <td>0.117897</td>\n",
       "      <td>0.897452</td>\n",
       "      <td>17.947474</td>\n",
       "      <td>124.360310</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td></td>\n",
       "      <td>11</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.041354</td>\n",
       "      <td>0.591452</td>\n",
       "      <td>1.913189</td>\n",
       "      <td>0.057195</td>\n",
       "      <td>0.063683</td>\n",
       "      <td>0.185009</td>\n",
       "      <td>0.182085</td>\n",
       "      <td>0.059142</td>\n",
       "      <td>0.956595</td>\n",
       "      <td>-40.854837</td>\n",
       "      <td>91.318929</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td></td>\n",
       "      <td>12</td>\n",
       "      <td>0.600007</td>\n",
       "      <td>0.019753</td>\n",
       "      <td>0.253448</td>\n",
       "      <td>1.636548</td>\n",
       "      <td>0.024509</td>\n",
       "      <td>0.029184</td>\n",
       "      <td>0.158257</td>\n",
       "      <td>0.156600</td>\n",
       "      <td>0.025347</td>\n",
       "      <td>0.981941</td>\n",
       "      <td>-74.655234</td>\n",
       "      <td>63.654844</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td></td>\n",
       "      <td>13</td>\n",
       "      <td>0.699996</td>\n",
       "      <td>0.009851</td>\n",
       "      <td>0.114815</td>\n",
       "      <td>1.419181</td>\n",
       "      <td>0.011103</td>\n",
       "      <td>0.014177</td>\n",
       "      <td>0.137238</td>\n",
       "      <td>0.136256</td>\n",
       "      <td>0.011480</td>\n",
       "      <td>0.993421</td>\n",
       "      <td>-88.518543</td>\n",
       "      <td>41.918113</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td></td>\n",
       "      <td>14</td>\n",
       "      <td>0.800004</td>\n",
       "      <td>0.005628</td>\n",
       "      <td>0.047078</td>\n",
       "      <td>1.247656</td>\n",
       "      <td>0.004553</td>\n",
       "      <td>0.007436</td>\n",
       "      <td>0.120651</td>\n",
       "      <td>0.120152</td>\n",
       "      <td>0.004708</td>\n",
       "      <td>0.998130</td>\n",
       "      <td>-95.292194</td>\n",
       "      <td>24.765621</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td></td>\n",
       "      <td>15</td>\n",
       "      <td>0.900055</td>\n",
       "      <td>0.003055</td>\n",
       "      <td>0.016760</td>\n",
       "      <td>1.110828</td>\n",
       "      <td>0.001621</td>\n",
       "      <td>0.004307</td>\n",
       "      <td>0.107419</td>\n",
       "      <td>0.107275</td>\n",
       "      <td>0.001677</td>\n",
       "      <td>0.999807</td>\n",
       "      <td>-98.323979</td>\n",
       "      <td>11.082839</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td></td>\n",
       "      <td>16</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000084</td>\n",
       "      <td>0.001936</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000187</td>\n",
       "      <td>0.001854</td>\n",
       "      <td>0.096702</td>\n",
       "      <td>0.096738</td>\n",
       "      <td>0.000193</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-99.806408</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      group  cumulative_data_fraction  lower_threshold      lift  \\\n",
       "0         1                  0.010029         0.398797  6.283089   \n",
       "1         2                  0.020101         0.355230  4.354127   \n",
       "2         3                  0.030030         0.332043  3.585591   \n",
       "3         4                  0.040015         0.312552  3.352283   \n",
       "4         5                  0.050069         0.298603  3.406388   \n",
       "5         6                  0.100001         0.260014  2.839075   \n",
       "6         7                  0.150083         0.229153  2.601360   \n",
       "7         8                  0.200021         0.201311  2.332451   \n",
       "8         9                  0.300047         0.147321  1.807975   \n",
       "9        10                  0.400005         0.090062  1.179475   \n",
       "10       11                  0.500000         0.041354  0.591452   \n",
       "11       12                  0.600007         0.019753  0.253448   \n",
       "12       13                  0.699996         0.009851  0.114815   \n",
       "13       14                  0.800004         0.005628  0.047078   \n",
       "14       15                  0.900055         0.003055  0.016760   \n",
       "15       16                  1.000000         0.000084  0.001936   \n",
       "\n",
       "    cumulative_lift  response_rate     score  cumulative_response_rate  \\\n",
       "0          6.283089       0.607587  0.504357                  0.607587   \n",
       "1          5.316513       0.421053  0.375510                  0.514117   \n",
       "2          4.744212       0.346734  0.344439                  0.458775   \n",
       "3          4.396881       0.324172  0.321365                  0.425187   \n",
       "4          4.197993       0.329404  0.305312                  0.405954   \n",
       "5          3.519466       0.274544  0.277976                  0.340339   \n",
       "6          3.213100       0.251557  0.243968                  0.310713   \n",
       "7          2.993233       0.225553  0.215210                  0.289452   \n",
       "8          2.598106       0.174835  0.174280                  0.251242   \n",
       "9          2.243603       0.114058  0.118626                  0.216961   \n",
       "10         1.913189       0.057195  0.063683                  0.185009   \n",
       "11         1.636548       0.024509  0.029184                  0.158257   \n",
       "12         1.419181       0.011103  0.014177                  0.137238   \n",
       "13         1.247656       0.004553  0.007436                  0.120651   \n",
       "14         1.110828       0.001621  0.004307                  0.107419   \n",
       "15         1.000000       0.000187  0.001854                  0.096702   \n",
       "\n",
       "    cumulative_score  capture_rate  cumulative_capture_rate        gain  \\\n",
       "0           0.504357      0.063012                 0.063012  528.308899   \n",
       "1           0.439794      0.043857                 0.106869  335.412685   \n",
       "2           0.408267      0.035601                 0.142470  258.559064   \n",
       "3           0.386582      0.033473                 0.175943  235.228333   \n",
       "4           0.370263      0.034247                 0.210190  240.638848   \n",
       "5           0.324183      0.141761                 0.351951  183.907453   \n",
       "6           0.297416      0.130281                 0.482232  160.136035   \n",
       "7           0.276892      0.116479                 0.598710  133.245143   \n",
       "8           0.242684      0.180845                 0.779555   80.797529   \n",
       "9           0.211683      0.117897                 0.897452   17.947474   \n",
       "10          0.182085      0.059142                 0.956595  -40.854837   \n",
       "11          0.156600      0.025347                 0.981941  -74.655234   \n",
       "12          0.136256      0.011480                 0.993421  -88.518543   \n",
       "13          0.120152      0.004708                 0.998130  -95.292194   \n",
       "14          0.107275      0.001677                 0.999807  -98.323979   \n",
       "15          0.096738      0.000193                 1.000000  -99.806408   \n",
       "\n",
       "    cumulative_gain  \n",
       "0        528.308899  \n",
       "1        431.651317  \n",
       "2        374.421231  \n",
       "3        339.688090  \n",
       "4        319.799279  \n",
       "5        251.946593  \n",
       "6        221.310008  \n",
       "7        199.323311  \n",
       "8        159.810611  \n",
       "9        124.360310  \n",
       "10        91.318929  \n",
       "11        63.654844  \n",
       "12        41.918113  \n",
       "13        24.765621  \n",
       "14        11.082839  \n",
       "15         0.000000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "ModelMetricsBinomial: xgboost\n",
      "** Reported on cross-validation data. **\n",
      "\n",
      "MSE: 0.07591257519788946\n",
      "RMSE: 0.27552236787217377\n",
      "LogLoss: 0.25259123052519633\n",
      "Mean Per-Class Error: 0.23773409899953646\n",
      "AUC: 0.8238216494036668\n",
      "pr_auc: 0.29450270841790577\n",
      "Gini: 0.6476432988073335\n",
      "\n",
      "Confusion Matrix (Act/Pred) for max f1 @ threshold = 0.18051382249051875: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>Error</th>\n",
       "      <th>Rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>116129.0</td>\n",
       "      <td>28704.0</td>\n",
       "      <td>0.1982</td>\n",
       "      <td>(28704.0/144833.0)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>5490.0</td>\n",
       "      <td>10015.0</td>\n",
       "      <td>0.3541</td>\n",
       "      <td>(5490.0/15505.0)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Total</td>\n",
       "      <td>121619.0</td>\n",
       "      <td>38719.0</td>\n",
       "      <td>0.2133</td>\n",
       "      <td>(34194.0/160338.0)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 0        1   Error                 Rate\n",
       "0      0  116129.0  28704.0  0.1982   (28704.0/144833.0)\n",
       "1      1    5490.0  10015.0  0.3541     (5490.0/15505.0)\n",
       "2  Total  121619.0  38719.0  0.2133   (34194.0/160338.0)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Maximum Metrics: Maximum metrics at their respective thresholds\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>metric</th>\n",
       "      <th>threshold</th>\n",
       "      <th>value</th>\n",
       "      <th>idx</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>max f1</td>\n",
       "      <td>0.180514</td>\n",
       "      <td>0.369394</td>\n",
       "      <td>214.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>max f2</td>\n",
       "      <td>0.108680</td>\n",
       "      <td>0.538509</td>\n",
       "      <td>275.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>max f0point5</td>\n",
       "      <td>0.267184</td>\n",
       "      <td>0.312990</td>\n",
       "      <td>142.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>max accuracy</td>\n",
       "      <td>0.647777</td>\n",
       "      <td>0.903847</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>max precision</td>\n",
       "      <td>0.984118</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>max recall</td>\n",
       "      <td>0.000463</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>399.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>max specificity</td>\n",
       "      <td>0.984118</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>max absolute_mcc</td>\n",
       "      <td>0.124495</td>\n",
       "      <td>0.321406</td>\n",
       "      <td>262.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>max min_per_class_accuracy</td>\n",
       "      <td>0.148132</td>\n",
       "      <td>0.747049</td>\n",
       "      <td>242.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>max mean_per_class_accuracy</td>\n",
       "      <td>0.096345</td>\n",
       "      <td>0.762266</td>\n",
       "      <td>284.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>max tns</td>\n",
       "      <td>0.984118</td>\n",
       "      <td>144833.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>max fns</td>\n",
       "      <td>0.984118</td>\n",
       "      <td>15504.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>max fps</td>\n",
       "      <td>0.000463</td>\n",
       "      <td>144833.000000</td>\n",
       "      <td>399.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>max tps</td>\n",
       "      <td>0.000463</td>\n",
       "      <td>15505.000000</td>\n",
       "      <td>399.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>max tnr</td>\n",
       "      <td>0.984118</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>max fnr</td>\n",
       "      <td>0.984118</td>\n",
       "      <td>0.999936</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>max fpr</td>\n",
       "      <td>0.000463</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>399.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>max tpr</td>\n",
       "      <td>0.000463</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>399.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         metric  threshold          value    idx\n",
       "0                        max f1   0.180514       0.369394  214.0\n",
       "1                        max f2   0.108680       0.538509  275.0\n",
       "2                  max f0point5   0.267184       0.312990  142.0\n",
       "3                  max accuracy   0.647777       0.903847   24.0\n",
       "4                 max precision   0.984118       1.000000    0.0\n",
       "5                    max recall   0.000463       1.000000  399.0\n",
       "6               max specificity   0.984118       1.000000    0.0\n",
       "7              max absolute_mcc   0.124495       0.321406  262.0\n",
       "8    max min_per_class_accuracy   0.148132       0.747049  242.0\n",
       "9   max mean_per_class_accuracy   0.096345       0.762266  284.0\n",
       "10                      max tns   0.984118  144833.000000    0.0\n",
       "11                      max fns   0.984118   15504.000000    0.0\n",
       "12                      max fps   0.000463  144833.000000  399.0\n",
       "13                      max tps   0.000463   15505.000000  399.0\n",
       "14                      max tnr   0.984118       1.000000    0.0\n",
       "15                      max fnr   0.984118       0.999936    0.0\n",
       "16                      max fpr   0.000463       1.000000  399.0\n",
       "17                      max tpr   0.000463       1.000000  399.0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Gains/Lift Table: Avg response rate:  9.67 %, avg score:  9.65 %\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>group</th>\n",
       "      <th>cumulative_data_fraction</th>\n",
       "      <th>lower_threshold</th>\n",
       "      <th>lift</th>\n",
       "      <th>cumulative_lift</th>\n",
       "      <th>response_rate</th>\n",
       "      <th>score</th>\n",
       "      <th>cumulative_response_rate</th>\n",
       "      <th>cumulative_score</th>\n",
       "      <th>capture_rate</th>\n",
       "      <th>cumulative_capture_rate</th>\n",
       "      <th>gain</th>\n",
       "      <th>cumulative_gain</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td></td>\n",
       "      <td>1</td>\n",
       "      <td>0.010004</td>\n",
       "      <td>0.407604</td>\n",
       "      <td>4.848174</td>\n",
       "      <td>4.848174</td>\n",
       "      <td>0.468828</td>\n",
       "      <td>0.503449</td>\n",
       "      <td>0.468828</td>\n",
       "      <td>0.503449</td>\n",
       "      <td>0.048500</td>\n",
       "      <td>0.048500</td>\n",
       "      <td>384.817366</td>\n",
       "      <td>384.817366</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td></td>\n",
       "      <td>2</td>\n",
       "      <td>0.020001</td>\n",
       "      <td>0.357424</td>\n",
       "      <td>3.760969</td>\n",
       "      <td>4.304741</td>\n",
       "      <td>0.363693</td>\n",
       "      <td>0.379752</td>\n",
       "      <td>0.416277</td>\n",
       "      <td>0.441620</td>\n",
       "      <td>0.037601</td>\n",
       "      <td>0.086101</td>\n",
       "      <td>276.096874</td>\n",
       "      <td>330.474071</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td></td>\n",
       "      <td>3</td>\n",
       "      <td>0.030005</td>\n",
       "      <td>0.331201</td>\n",
       "      <td>3.275096</td>\n",
       "      <td>3.961454</td>\n",
       "      <td>0.316708</td>\n",
       "      <td>0.343188</td>\n",
       "      <td>0.383080</td>\n",
       "      <td>0.408802</td>\n",
       "      <td>0.032764</td>\n",
       "      <td>0.118865</td>\n",
       "      <td>227.509604</td>\n",
       "      <td>296.145448</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td></td>\n",
       "      <td>4</td>\n",
       "      <td>0.040003</td>\n",
       "      <td>0.314332</td>\n",
       "      <td>3.173922</td>\n",
       "      <td>3.764633</td>\n",
       "      <td>0.306925</td>\n",
       "      <td>0.322009</td>\n",
       "      <td>0.364047</td>\n",
       "      <td>0.387111</td>\n",
       "      <td>0.031732</td>\n",
       "      <td>0.150597</td>\n",
       "      <td>217.392216</td>\n",
       "      <td>276.463279</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td></td>\n",
       "      <td>5</td>\n",
       "      <td>0.050001</td>\n",
       "      <td>0.300628</td>\n",
       "      <td>3.122314</td>\n",
       "      <td>3.636201</td>\n",
       "      <td>0.301934</td>\n",
       "      <td>0.307055</td>\n",
       "      <td>0.351628</td>\n",
       "      <td>0.371104</td>\n",
       "      <td>0.031216</td>\n",
       "      <td>0.181812</td>\n",
       "      <td>212.231367</td>\n",
       "      <td>263.620102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td></td>\n",
       "      <td>6</td>\n",
       "      <td>0.100020</td>\n",
       "      <td>0.256860</td>\n",
       "      <td>2.783832</td>\n",
       "      <td>3.209937</td>\n",
       "      <td>0.269202</td>\n",
       "      <td>0.277189</td>\n",
       "      <td>0.310407</td>\n",
       "      <td>0.324137</td>\n",
       "      <td>0.139245</td>\n",
       "      <td>0.321058</td>\n",
       "      <td>178.383163</td>\n",
       "      <td>220.993660</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td></td>\n",
       "      <td>7</td>\n",
       "      <td>0.150008</td>\n",
       "      <td>0.227910</td>\n",
       "      <td>2.514624</td>\n",
       "      <td>2.978233</td>\n",
       "      <td>0.243169</td>\n",
       "      <td>0.242128</td>\n",
       "      <td>0.288001</td>\n",
       "      <td>0.296809</td>\n",
       "      <td>0.125701</td>\n",
       "      <td>0.446759</td>\n",
       "      <td>151.462370</td>\n",
       "      <td>197.823309</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td></td>\n",
       "      <td>8</td>\n",
       "      <td>0.200002</td>\n",
       "      <td>0.200990</td>\n",
       "      <td>2.293711</td>\n",
       "      <td>2.807124</td>\n",
       "      <td>0.221806</td>\n",
       "      <td>0.214426</td>\n",
       "      <td>0.271454</td>\n",
       "      <td>0.276216</td>\n",
       "      <td>0.114673</td>\n",
       "      <td>0.561432</td>\n",
       "      <td>129.371122</td>\n",
       "      <td>180.712397</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td></td>\n",
       "      <td>9</td>\n",
       "      <td>0.300004</td>\n",
       "      <td>0.146988</td>\n",
       "      <td>1.865826</td>\n",
       "      <td>2.493358</td>\n",
       "      <td>0.180429</td>\n",
       "      <td>0.174819</td>\n",
       "      <td>0.241113</td>\n",
       "      <td>0.242417</td>\n",
       "      <td>0.186585</td>\n",
       "      <td>0.748017</td>\n",
       "      <td>86.582645</td>\n",
       "      <td>149.335813</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td></td>\n",
       "      <td>10</td>\n",
       "      <td>0.399999</td>\n",
       "      <td>0.089371</td>\n",
       "      <td>1.240952</td>\n",
       "      <td>2.180271</td>\n",
       "      <td>0.120002</td>\n",
       "      <td>0.118056</td>\n",
       "      <td>0.210837</td>\n",
       "      <td>0.211328</td>\n",
       "      <td>0.124089</td>\n",
       "      <td>0.872106</td>\n",
       "      <td>24.095195</td>\n",
       "      <td>118.027123</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td></td>\n",
       "      <td>11</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.041830</td>\n",
       "      <td>0.669453</td>\n",
       "      <td>1.878104</td>\n",
       "      <td>0.064737</td>\n",
       "      <td>0.063571</td>\n",
       "      <td>0.181616</td>\n",
       "      <td>0.181776</td>\n",
       "      <td>0.066946</td>\n",
       "      <td>0.939052</td>\n",
       "      <td>-33.054689</td>\n",
       "      <td>87.810384</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td></td>\n",
       "      <td>12</td>\n",
       "      <td>0.600001</td>\n",
       "      <td>0.019364</td>\n",
       "      <td>0.306349</td>\n",
       "      <td>1.616142</td>\n",
       "      <td>0.029625</td>\n",
       "      <td>0.029221</td>\n",
       "      <td>0.156284</td>\n",
       "      <td>0.156350</td>\n",
       "      <td>0.030635</td>\n",
       "      <td>0.969687</td>\n",
       "      <td>-69.365103</td>\n",
       "      <td>61.614197</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td></td>\n",
       "      <td>13</td>\n",
       "      <td>0.699996</td>\n",
       "      <td>0.009619</td>\n",
       "      <td>0.151572</td>\n",
       "      <td>1.406927</td>\n",
       "      <td>0.014657</td>\n",
       "      <td>0.013857</td>\n",
       "      <td>0.136053</td>\n",
       "      <td>0.135995</td>\n",
       "      <td>0.015156</td>\n",
       "      <td>0.984844</td>\n",
       "      <td>-84.842843</td>\n",
       "      <td>40.692695</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td></td>\n",
       "      <td>14</td>\n",
       "      <td>0.799998</td>\n",
       "      <td>0.005469</td>\n",
       "      <td>0.083198</td>\n",
       "      <td>1.241458</td>\n",
       "      <td>0.008045</td>\n",
       "      <td>0.007282</td>\n",
       "      <td>0.120051</td>\n",
       "      <td>0.119906</td>\n",
       "      <td>0.008320</td>\n",
       "      <td>0.993163</td>\n",
       "      <td>-91.680207</td>\n",
       "      <td>24.145824</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td></td>\n",
       "      <td>15</td>\n",
       "      <td>0.899999</td>\n",
       "      <td>0.003041</td>\n",
       "      <td>0.050306</td>\n",
       "      <td>1.109106</td>\n",
       "      <td>0.004865</td>\n",
       "      <td>0.004200</td>\n",
       "      <td>0.107253</td>\n",
       "      <td>0.107049</td>\n",
       "      <td>0.005031</td>\n",
       "      <td>0.998194</td>\n",
       "      <td>-94.969427</td>\n",
       "      <td>10.910613</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td></td>\n",
       "      <td>16</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000058</td>\n",
       "      <td>0.018058</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.001746</td>\n",
       "      <td>0.001841</td>\n",
       "      <td>0.096702</td>\n",
       "      <td>0.096528</td>\n",
       "      <td>0.001806</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-98.194153</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      group  cumulative_data_fraction  lower_threshold      lift  \\\n",
       "0         1                  0.010004         0.407604  4.848174   \n",
       "1         2                  0.020001         0.357424  3.760969   \n",
       "2         3                  0.030005         0.331201  3.275096   \n",
       "3         4                  0.040003         0.314332  3.173922   \n",
       "4         5                  0.050001         0.300628  3.122314   \n",
       "5         6                  0.100020         0.256860  2.783832   \n",
       "6         7                  0.150008         0.227910  2.514624   \n",
       "7         8                  0.200002         0.200990  2.293711   \n",
       "8         9                  0.300004         0.146988  1.865826   \n",
       "9        10                  0.399999         0.089371  1.240952   \n",
       "10       11                  0.500000         0.041830  0.669453   \n",
       "11       12                  0.600001         0.019364  0.306349   \n",
       "12       13                  0.699996         0.009619  0.151572   \n",
       "13       14                  0.799998         0.005469  0.083198   \n",
       "14       15                  0.899999         0.003041  0.050306   \n",
       "15       16                  1.000000         0.000058  0.018058   \n",
       "\n",
       "    cumulative_lift  response_rate     score  cumulative_response_rate  \\\n",
       "0          4.848174       0.468828  0.503449                  0.468828   \n",
       "1          4.304741       0.363693  0.379752                  0.416277   \n",
       "2          3.961454       0.316708  0.343188                  0.383080   \n",
       "3          3.764633       0.306925  0.322009                  0.364047   \n",
       "4          3.636201       0.301934  0.307055                  0.351628   \n",
       "5          3.209937       0.269202  0.277189                  0.310407   \n",
       "6          2.978233       0.243169  0.242128                  0.288001   \n",
       "7          2.807124       0.221806  0.214426                  0.271454   \n",
       "8          2.493358       0.180429  0.174819                  0.241113   \n",
       "9          2.180271       0.120002  0.118056                  0.210837   \n",
       "10         1.878104       0.064737  0.063571                  0.181616   \n",
       "11         1.616142       0.029625  0.029221                  0.156284   \n",
       "12         1.406927       0.014657  0.013857                  0.136053   \n",
       "13         1.241458       0.008045  0.007282                  0.120051   \n",
       "14         1.109106       0.004865  0.004200                  0.107253   \n",
       "15         1.000000       0.001746  0.001841                  0.096702   \n",
       "\n",
       "    cumulative_score  capture_rate  cumulative_capture_rate        gain  \\\n",
       "0           0.503449      0.048500                 0.048500  384.817366   \n",
       "1           0.441620      0.037601                 0.086101  276.096874   \n",
       "2           0.408802      0.032764                 0.118865  227.509604   \n",
       "3           0.387111      0.031732                 0.150597  217.392216   \n",
       "4           0.371104      0.031216                 0.181812  212.231367   \n",
       "5           0.324137      0.139245                 0.321058  178.383163   \n",
       "6           0.296809      0.125701                 0.446759  151.462370   \n",
       "7           0.276216      0.114673                 0.561432  129.371122   \n",
       "8           0.242417      0.186585                 0.748017   86.582645   \n",
       "9           0.211328      0.124089                 0.872106   24.095195   \n",
       "10          0.181776      0.066946                 0.939052  -33.054689   \n",
       "11          0.156350      0.030635                 0.969687  -69.365103   \n",
       "12          0.135995      0.015156                 0.984844  -84.842843   \n",
       "13          0.119906      0.008320                 0.993163  -91.680207   \n",
       "14          0.107049      0.005031                 0.998194  -94.969427   \n",
       "15          0.096528      0.001806                 1.000000  -98.194153   \n",
       "\n",
       "    cumulative_gain  \n",
       "0        384.817366  \n",
       "1        330.474071  \n",
       "2        296.145448  \n",
       "3        276.463279  \n",
       "4        263.620102  \n",
       "5        220.993660  \n",
       "6        197.823309  \n",
       "7        180.712397  \n",
       "8        149.335813  \n",
       "9        118.027123  \n",
       "10        87.810384  \n",
       "11        61.614197  \n",
       "12        40.692695  \n",
       "13        24.145824  \n",
       "14        10.910613  \n",
       "15         0.000000  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Cross-Validation Metrics Summary: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>sd</th>\n",
       "      <th>cv_1_valid</th>\n",
       "      <th>cv_2_valid</th>\n",
       "      <th>cv_3_valid</th>\n",
       "      <th>cv_4_valid</th>\n",
       "      <th>cv_5_valid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>accuracy</td>\n",
       "      <td>0.7828028</td>\n",
       "      <td>0.014824913</td>\n",
       "      <td>0.7890112</td>\n",
       "      <td>0.7881207</td>\n",
       "      <td>0.7686139</td>\n",
       "      <td>0.8015615</td>\n",
       "      <td>0.766707</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>auc</td>\n",
       "      <td>0.8239214</td>\n",
       "      <td>0.0030833608</td>\n",
       "      <td>0.82043314</td>\n",
       "      <td>0.8278595</td>\n",
       "      <td>0.82220495</td>\n",
       "      <td>0.8263717</td>\n",
       "      <td>0.8227375</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>err</td>\n",
       "      <td>0.21719715</td>\n",
       "      <td>0.014824913</td>\n",
       "      <td>0.2109888</td>\n",
       "      <td>0.21187933</td>\n",
       "      <td>0.23138613</td>\n",
       "      <td>0.19843851</td>\n",
       "      <td>0.23329298</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>err_count</td>\n",
       "      <td>6965.0</td>\n",
       "      <td>478.78857</td>\n",
       "      <td>6747.0</td>\n",
       "      <td>6742.0</td>\n",
       "      <td>7415.0</td>\n",
       "      <td>6405.0</td>\n",
       "      <td>7516.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>f0point5</td>\n",
       "      <td>0.29361784</td>\n",
       "      <td>0.009722157</td>\n",
       "      <td>0.2951657</td>\n",
       "      <td>0.30725196</td>\n",
       "      <td>0.27999464</td>\n",
       "      <td>0.294061</td>\n",
       "      <td>0.29161587</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>f1</td>\n",
       "      <td>0.37064147</td>\n",
       "      <td>0.009523416</td>\n",
       "      <td>0.37079176</td>\n",
       "      <td>0.38429224</td>\n",
       "      <td>0.35983768</td>\n",
       "      <td>0.36388916</td>\n",
       "      <td>0.37439653</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>f2</td>\n",
       "      <td>0.50296193</td>\n",
       "      <td>0.017143423</td>\n",
       "      <td>0.4985205</td>\n",
       "      <td>0.5128955</td>\n",
       "      <td>0.50338167</td>\n",
       "      <td>0.4772076</td>\n",
       "      <td>0.5228044</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>lift_top_group</td>\n",
       "      <td>4.8096123</td>\n",
       "      <td>0.29514456</td>\n",
       "      <td>4.506554</td>\n",
       "      <td>4.9452076</td>\n",
       "      <td>4.927531</td>\n",
       "      <td>4.500409</td>\n",
       "      <td>5.1683593</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>logloss</td>\n",
       "      <td>0.25260153</td>\n",
       "      <td>0.004006559</td>\n",
       "      <td>0.253692</td>\n",
       "      <td>0.25633025</td>\n",
       "      <td>0.25011146</td>\n",
       "      <td>0.24695344</td>\n",
       "      <td>0.2559206</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>max_per_class_error</td>\n",
       "      <td>0.33882216</td>\n",
       "      <td>0.040979568</td>\n",
       "      <td>0.3528646</td>\n",
       "      <td>0.339818</td>\n",
       "      <td>0.3142481</td>\n",
       "      <td>0.39776462</td>\n",
       "      <td>0.28941548</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>mcc</td>\n",
       "      <td>0.3129418</td>\n",
       "      <td>0.009861303</td>\n",
       "      <td>0.31166923</td>\n",
       "      <td>0.32332623</td>\n",
       "      <td>0.30680245</td>\n",
       "      <td>0.30056018</td>\n",
       "      <td>0.3223508</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>mean_per_class_accuracy</td>\n",
       "      <td>0.72847575</td>\n",
       "      <td>0.010757886</td>\n",
       "      <td>0.7256123</td>\n",
       "      <td>0.73127145</td>\n",
       "      <td>0.7315235</td>\n",
       "      <td>0.7122687</td>\n",
       "      <td>0.74170285</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>mean_per_class_error</td>\n",
       "      <td>0.27152425</td>\n",
       "      <td>0.010757886</td>\n",
       "      <td>0.27438775</td>\n",
       "      <td>0.26872855</td>\n",
       "      <td>0.2684765</td>\n",
       "      <td>0.2877313</td>\n",
       "      <td>0.25829718</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>mse</td>\n",
       "      <td>0.07591619</td>\n",
       "      <td>0.0012561207</td>\n",
       "      <td>0.075878546</td>\n",
       "      <td>0.07754413</td>\n",
       "      <td>0.07505254</td>\n",
       "      <td>0.074401826</td>\n",
       "      <td>0.07670395</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>precision</td>\n",
       "      <td>0.257928</td>\n",
       "      <td>0.009920426</td>\n",
       "      <td>0.2598353</td>\n",
       "      <td>0.27102923</td>\n",
       "      <td>0.24391386</td>\n",
       "      <td>0.2607087</td>\n",
       "      <td>0.254153</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>r2</td>\n",
       "      <td>0.13080788</td>\n",
       "      <td>0.005957356</td>\n",
       "      <td>0.12619933</td>\n",
       "      <td>0.13960011</td>\n",
       "      <td>0.1256617</td>\n",
       "      <td>0.12841928</td>\n",
       "      <td>0.13415897</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>recall</td>\n",
       "      <td>0.6611778</td>\n",
       "      <td>0.040979568</td>\n",
       "      <td>0.64713544</td>\n",
       "      <td>0.660182</td>\n",
       "      <td>0.6857519</td>\n",
       "      <td>0.6022354</td>\n",
       "      <td>0.7105845</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>rmse</td>\n",
       "      <td>0.2755214</td>\n",
       "      <td>0.0022787692</td>\n",
       "      <td>0.2754606</td>\n",
       "      <td>0.27846745</td>\n",
       "      <td>0.27395716</td>\n",
       "      <td>0.27276698</td>\n",
       "      <td>0.27695477</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>specificity</td>\n",
       "      <td>0.7957737</td>\n",
       "      <td>0.02052174</td>\n",
       "      <td>0.8040891</td>\n",
       "      <td>0.8023609</td>\n",
       "      <td>0.7772951</td>\n",
       "      <td>0.82230204</td>\n",
       "      <td>0.7728211</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   mean            sd   cv_1_valid  \\\n",
       "0                  accuracy   0.7828028   0.014824913    0.7890112   \n",
       "1                       auc   0.8239214  0.0030833608   0.82043314   \n",
       "2                       err  0.21719715   0.014824913    0.2109888   \n",
       "3                 err_count      6965.0     478.78857       6747.0   \n",
       "4                  f0point5  0.29361784   0.009722157    0.2951657   \n",
       "5                        f1  0.37064147   0.009523416   0.37079176   \n",
       "6                        f2  0.50296193   0.017143423    0.4985205   \n",
       "7            lift_top_group   4.8096123    0.29514456     4.506554   \n",
       "8                   logloss  0.25260153   0.004006559     0.253692   \n",
       "9       max_per_class_error  0.33882216   0.040979568    0.3528646   \n",
       "10                      mcc   0.3129418   0.009861303   0.31166923   \n",
       "11  mean_per_class_accuracy  0.72847575   0.010757886    0.7256123   \n",
       "12     mean_per_class_error  0.27152425   0.010757886   0.27438775   \n",
       "13                      mse  0.07591619  0.0012561207  0.075878546   \n",
       "14                precision    0.257928   0.009920426    0.2598353   \n",
       "15                       r2  0.13080788   0.005957356   0.12619933   \n",
       "16                   recall   0.6611778   0.040979568   0.64713544   \n",
       "17                     rmse   0.2755214  0.0022787692    0.2754606   \n",
       "18              specificity   0.7957737    0.02052174    0.8040891   \n",
       "\n",
       "    cv_2_valid  cv_3_valid   cv_4_valid  cv_5_valid  \n",
       "0    0.7881207   0.7686139    0.8015615    0.766707  \n",
       "1    0.8278595  0.82220495    0.8263717   0.8227375  \n",
       "2   0.21187933  0.23138613   0.19843851  0.23329298  \n",
       "3       6742.0      7415.0       6405.0      7516.0  \n",
       "4   0.30725196  0.27999464     0.294061  0.29161587  \n",
       "5   0.38429224  0.35983768   0.36388916  0.37439653  \n",
       "6    0.5128955  0.50338167    0.4772076   0.5228044  \n",
       "7    4.9452076    4.927531     4.500409   5.1683593  \n",
       "8   0.25633025  0.25011146   0.24695344   0.2559206  \n",
       "9     0.339818   0.3142481   0.39776462  0.28941548  \n",
       "10  0.32332623  0.30680245   0.30056018   0.3223508  \n",
       "11  0.73127145   0.7315235    0.7122687  0.74170285  \n",
       "12  0.26872855   0.2684765    0.2877313  0.25829718  \n",
       "13  0.07754413  0.07505254  0.074401826  0.07670395  \n",
       "14  0.27102923  0.24391386    0.2607087    0.254153  \n",
       "15  0.13960011   0.1256617   0.12841928  0.13415897  \n",
       "16    0.660182   0.6857519    0.6022354   0.7105845  \n",
       "17  0.27846745  0.27395716   0.27276698  0.27695477  \n",
       "18   0.8023609   0.7772951   0.82230204   0.7728211  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Scoring History: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>timestamp</th>\n",
       "      <th>duration</th>\n",
       "      <th>number_of_trees</th>\n",
       "      <th>training_rmse</th>\n",
       "      <th>training_logloss</th>\n",
       "      <th>training_auc</th>\n",
       "      <th>training_pr_auc</th>\n",
       "      <th>training_lift</th>\n",
       "      <th>training_classification_error</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:22:27</td>\n",
       "      <td>23.165 sec</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.693147</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.903298</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:22:27</td>\n",
       "      <td>23.297 sec</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.404391</td>\n",
       "      <td>0.516382</td>\n",
       "      <td>0.809869</td>\n",
       "      <td>0.280465</td>\n",
       "      <td>4.499264</td>\n",
       "      <td>0.250932</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:22:27</td>\n",
       "      <td>23.359 sec</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.348688</td>\n",
       "      <td>0.420389</td>\n",
       "      <td>0.814731</td>\n",
       "      <td>0.288584</td>\n",
       "      <td>4.714393</td>\n",
       "      <td>0.205566</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:22:27</td>\n",
       "      <td>23.420 sec</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.316417</td>\n",
       "      <td>0.362362</td>\n",
       "      <td>0.817815</td>\n",
       "      <td>0.292598</td>\n",
       "      <td>4.594236</td>\n",
       "      <td>0.198543</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:22:27</td>\n",
       "      <td>23.479 sec</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.298103</td>\n",
       "      <td>0.325531</td>\n",
       "      <td>0.819057</td>\n",
       "      <td>0.295706</td>\n",
       "      <td>4.663219</td>\n",
       "      <td>0.231636</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:22:27</td>\n",
       "      <td>23.560 sec</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.287869</td>\n",
       "      <td>0.301381</td>\n",
       "      <td>0.820475</td>\n",
       "      <td>0.297663</td>\n",
       "      <td>4.988231</td>\n",
       "      <td>0.201524</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:22:27</td>\n",
       "      <td>23.626 sec</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.282221</td>\n",
       "      <td>0.285339</td>\n",
       "      <td>0.821568</td>\n",
       "      <td>0.299208</td>\n",
       "      <td>5.036385</td>\n",
       "      <td>0.202154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:22:27</td>\n",
       "      <td>23.699 sec</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.279034</td>\n",
       "      <td>0.274409</td>\n",
       "      <td>0.822936</td>\n",
       "      <td>0.300948</td>\n",
       "      <td>5.023653</td>\n",
       "      <td>0.220659</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:22:27</td>\n",
       "      <td>23.774 sec</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.277181</td>\n",
       "      <td>0.266922</td>\n",
       "      <td>0.823907</td>\n",
       "      <td>0.303666</td>\n",
       "      <td>5.151814</td>\n",
       "      <td>0.212077</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:22:28</td>\n",
       "      <td>23.855 sec</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.276026</td>\n",
       "      <td>0.261623</td>\n",
       "      <td>0.825572</td>\n",
       "      <td>0.306710</td>\n",
       "      <td>5.303103</td>\n",
       "      <td>0.203994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:22:28</td>\n",
       "      <td>23.938 sec</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.275385</td>\n",
       "      <td>0.258041</td>\n",
       "      <td>0.826688</td>\n",
       "      <td>0.308418</td>\n",
       "      <td>5.285703</td>\n",
       "      <td>0.215114</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:22:28</td>\n",
       "      <td>24.030 sec</td>\n",
       "      <td>11.0</td>\n",
       "      <td>0.274871</td>\n",
       "      <td>0.255371</td>\n",
       "      <td>0.827796</td>\n",
       "      <td>0.311076</td>\n",
       "      <td>5.388999</td>\n",
       "      <td>0.205603</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:22:28</td>\n",
       "      <td>24.123 sec</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.274542</td>\n",
       "      <td>0.253553</td>\n",
       "      <td>0.828686</td>\n",
       "      <td>0.312538</td>\n",
       "      <td>5.344379</td>\n",
       "      <td>0.204948</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:22:28</td>\n",
       "      <td>24.206 sec</td>\n",
       "      <td>13.0</td>\n",
       "      <td>0.274309</td>\n",
       "      <td>0.252175</td>\n",
       "      <td>0.829578</td>\n",
       "      <td>0.313802</td>\n",
       "      <td>5.363456</td>\n",
       "      <td>0.204580</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:22:28</td>\n",
       "      <td>24.293 sec</td>\n",
       "      <td>14.0</td>\n",
       "      <td>0.274165</td>\n",
       "      <td>0.251213</td>\n",
       "      <td>0.830095</td>\n",
       "      <td>0.314673</td>\n",
       "      <td>5.439001</td>\n",
       "      <td>0.206034</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:22:28</td>\n",
       "      <td>24.383 sec</td>\n",
       "      <td>15.0</td>\n",
       "      <td>0.273938</td>\n",
       "      <td>0.250265</td>\n",
       "      <td>0.831029</td>\n",
       "      <td>0.314919</td>\n",
       "      <td>5.524231</td>\n",
       "      <td>0.201955</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:22:28</td>\n",
       "      <td>24.492 sec</td>\n",
       "      <td>16.0</td>\n",
       "      <td>0.273731</td>\n",
       "      <td>0.249563</td>\n",
       "      <td>0.831871</td>\n",
       "      <td>0.317455</td>\n",
       "      <td>5.578055</td>\n",
       "      <td>0.208971</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:22:28</td>\n",
       "      <td>24.616 sec</td>\n",
       "      <td>17.0</td>\n",
       "      <td>0.273597</td>\n",
       "      <td>0.248997</td>\n",
       "      <td>0.832512</td>\n",
       "      <td>0.319617</td>\n",
       "      <td>5.646420</td>\n",
       "      <td>0.203632</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:22:28</td>\n",
       "      <td>24.737 sec</td>\n",
       "      <td>18.0</td>\n",
       "      <td>0.273423</td>\n",
       "      <td>0.248434</td>\n",
       "      <td>0.833292</td>\n",
       "      <td>0.321005</td>\n",
       "      <td>5.692736</td>\n",
       "      <td>0.202934</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td></td>\n",
       "      <td>2019-12-09 13:22:29</td>\n",
       "      <td>24.851 sec</td>\n",
       "      <td>19.0</td>\n",
       "      <td>0.273315</td>\n",
       "      <td>0.248054</td>\n",
       "      <td>0.833883</td>\n",
       "      <td>0.322120</td>\n",
       "      <td>5.706514</td>\n",
       "      <td>0.205310</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                timestamp    duration  number_of_trees  training_rmse  \\\n",
       "0     2019-12-09 13:22:27  23.165 sec              0.0       0.500000   \n",
       "1     2019-12-09 13:22:27  23.297 sec              1.0       0.404391   \n",
       "2     2019-12-09 13:22:27  23.359 sec              2.0       0.348688   \n",
       "3     2019-12-09 13:22:27  23.420 sec              3.0       0.316417   \n",
       "4     2019-12-09 13:22:27  23.479 sec              4.0       0.298103   \n",
       "5     2019-12-09 13:22:27  23.560 sec              5.0       0.287869   \n",
       "6     2019-12-09 13:22:27  23.626 sec              6.0       0.282221   \n",
       "7     2019-12-09 13:22:27  23.699 sec              7.0       0.279034   \n",
       "8     2019-12-09 13:22:27  23.774 sec              8.0       0.277181   \n",
       "9     2019-12-09 13:22:28  23.855 sec              9.0       0.276026   \n",
       "10    2019-12-09 13:22:28  23.938 sec             10.0       0.275385   \n",
       "11    2019-12-09 13:22:28  24.030 sec             11.0       0.274871   \n",
       "12    2019-12-09 13:22:28  24.123 sec             12.0       0.274542   \n",
       "13    2019-12-09 13:22:28  24.206 sec             13.0       0.274309   \n",
       "14    2019-12-09 13:22:28  24.293 sec             14.0       0.274165   \n",
       "15    2019-12-09 13:22:28  24.383 sec             15.0       0.273938   \n",
       "16    2019-12-09 13:22:28  24.492 sec             16.0       0.273731   \n",
       "17    2019-12-09 13:22:28  24.616 sec             17.0       0.273597   \n",
       "18    2019-12-09 13:22:28  24.737 sec             18.0       0.273423   \n",
       "19    2019-12-09 13:22:29  24.851 sec             19.0       0.273315   \n",
       "\n",
       "    training_logloss  training_auc  training_pr_auc  training_lift  \\\n",
       "0           0.693147      0.500000         0.000000       1.000000   \n",
       "1           0.516382      0.809869         0.280465       4.499264   \n",
       "2           0.420389      0.814731         0.288584       4.714393   \n",
       "3           0.362362      0.817815         0.292598       4.594236   \n",
       "4           0.325531      0.819057         0.295706       4.663219   \n",
       "5           0.301381      0.820475         0.297663       4.988231   \n",
       "6           0.285339      0.821568         0.299208       5.036385   \n",
       "7           0.274409      0.822936         0.300948       5.023653   \n",
       "8           0.266922      0.823907         0.303666       5.151814   \n",
       "9           0.261623      0.825572         0.306710       5.303103   \n",
       "10          0.258041      0.826688         0.308418       5.285703   \n",
       "11          0.255371      0.827796         0.311076       5.388999   \n",
       "12          0.253553      0.828686         0.312538       5.344379   \n",
       "13          0.252175      0.829578         0.313802       5.363456   \n",
       "14          0.251213      0.830095         0.314673       5.439001   \n",
       "15          0.250265      0.831029         0.314919       5.524231   \n",
       "16          0.249563      0.831871         0.317455       5.578055   \n",
       "17          0.248997      0.832512         0.319617       5.646420   \n",
       "18          0.248434      0.833292         0.321005       5.692736   \n",
       "19          0.248054      0.833883         0.322120       5.706514   \n",
       "\n",
       "    training_classification_error  \n",
       "0                        0.903298  \n",
       "1                        0.250932  \n",
       "2                        0.205566  \n",
       "3                        0.198543  \n",
       "4                        0.231636  \n",
       "5                        0.201524  \n",
       "6                        0.202154  \n",
       "7                        0.220659  \n",
       "8                        0.212077  \n",
       "9                        0.203994  \n",
       "10                       0.215114  \n",
       "11                       0.205603  \n",
       "12                       0.204948  \n",
       "13                       0.204580  \n",
       "14                       0.206034  \n",
       "15                       0.201955  \n",
       "16                       0.208971  \n",
       "17                       0.203632  \n",
       "18                       0.202934  \n",
       "19                       0.205310  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "See the whole table with table.as_data_frame()\n",
      "\n",
      "Variable Importances: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>variable</th>\n",
       "      <th>relative_importance</th>\n",
       "      <th>scaled_importance</th>\n",
       "      <th>percentage</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>loan_to_value_ratio_std</td>\n",
       "      <td>21562.857422</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.517523</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>property_value_std</td>\n",
       "      <td>5853.233398</td>\n",
       "      <td>0.271450</td>\n",
       "      <td>0.140482</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>debt_to_income_ratio_std</td>\n",
       "      <td>3998.149902</td>\n",
       "      <td>0.185418</td>\n",
       "      <td>0.095958</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>loan_amount_std</td>\n",
       "      <td>3346.707275</td>\n",
       "      <td>0.155207</td>\n",
       "      <td>0.080323</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>income_std</td>\n",
       "      <td>2775.886230</td>\n",
       "      <td>0.128735</td>\n",
       "      <td>0.066623</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>intro_rate_period_std</td>\n",
       "      <td>2675.397705</td>\n",
       "      <td>0.124074</td>\n",
       "      <td>0.064211</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>no_intro_rate_period_std</td>\n",
       "      <td>1030.331787</td>\n",
       "      <td>0.047783</td>\n",
       "      <td>0.024729</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>term_360</td>\n",
       "      <td>269.373260</td>\n",
       "      <td>0.012492</td>\n",
       "      <td>0.006465</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>conforming</td>\n",
       "      <td>116.245148</td>\n",
       "      <td>0.005391</td>\n",
       "      <td>0.002790</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>debt_to_income_ratio_missing.False</td>\n",
       "      <td>37.325474</td>\n",
       "      <td>0.001731</td>\n",
       "      <td>0.000896</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             variable  relative_importance  scaled_importance  \\\n",
       "0             loan_to_value_ratio_std         21562.857422           1.000000   \n",
       "1                  property_value_std          5853.233398           0.271450   \n",
       "2            debt_to_income_ratio_std          3998.149902           0.185418   \n",
       "3                     loan_amount_std          3346.707275           0.155207   \n",
       "4                          income_std          2775.886230           0.128735   \n",
       "5               intro_rate_period_std          2675.397705           0.124074   \n",
       "6            no_intro_rate_period_std          1030.331787           0.047783   \n",
       "7                            term_360           269.373260           0.012492   \n",
       "8                          conforming           116.245148           0.005391   \n",
       "9  debt_to_income_ratio_missing.False            37.325474           0.001731   \n",
       "\n",
       "   percentage  \n",
       "0    0.517523  \n",
       "1    0.140482  \n",
       "2    0.095958  \n",
       "3    0.080323  \n",
       "4    0.066623  \n",
       "5    0.064211  \n",
       "6    0.024729  \n",
       "7    0.006465  \n",
       "8    0.002790  \n",
       "9    0.000896  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": []
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb_default"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8280878975747641\n"
     ]
    }
   ],
   "source": [
    "# Get the AUC on the test set for the default GBM\n",
    "perf_xgb_default = xgb_default.model_performance(test)\n",
    "print(perf_xgb_default.auc())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8238216494036668\n"
     ]
    }
   ],
   "source": [
    "# Get the AUC on 5 fold CV for the default GBM\n",
    "perf_cv_xgb_default = xgb_default.model_performance(xval=True)\n",
    "print(perf_cv_xgb_default.auc())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Variable Importance of MXGB and XGB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA9oAAAJTCAYAAAD3327jAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3XmYZFV9//H3BwZBFkERFTE6Boi4ACObyiYIGuO4gSBRowLGBXdcUWJEowIh0YgKiAu4RQ0qiiKgiMiIyiLLDIiigTE/RRE3ZBGUme/vj3saiqKquwfudA/wfj1PPVV17r3nnHuru5/+1Dn33lQVkiRJkiSpHyvNdgckSZIkSborMWhLkiRJktQjg7YkSZIkST0yaEuSJEmS1CODtiRJkiRJPTJoS5IkSZLUI4O2JOluIcl3k9zUQz2/SPKzZVh/oySV5KN3tG1JknTnYNCWJC0XST7TAubLp7HuN9q6u81E3+5q2pcIlWT72e7L8rasX3TcHSTZtX3+p06yzsQXPj8bKn9QktckOTnJ4iQ3Jvld+5185hTtrpLkn5N8M8lvkvylPX8jyb5J5izjfny29fGNY5avm+SKJDck2WzE8nsk2TvJV9rPyQ1Jrk9yeZIvtT6tPrTNnNbm8OPGdjyOTbLJsuzHTEvyrrvL7790Z7JMfwAlSVoGHwGeC/wzcMS4lZLMBXYFfgV8dTn257nAPZdj/dKd0WuB1wOXAacBVwJzgd2AJyY5rKreNLxRkgcDJwCbA78GvtaeHwA8BXgi8PIkT6+qK6bZl5cDOwD/luSUqlo4tPxoYH3gDcPLkjwC+CKwCfCHti+XA0uABwE7tn16d6tjWAHvHHi/DvAY4IXAs5JsN6I/kjSWQVuStFxU1elJLgUenWSLqjpvzKovAgIcU1V3eGr3JP35v+VVt3Qn9gNgx6paMFiY5FHA94A3JvlMVV04sGxN4GTg4cDHgVdW1Z8Hlq8BHAX8E/D1JI8bXD5OVf0hyT7AKcCnk2xdVTe2OvcGdgdOB9471NcNgG/Rhfz3AW+rquuG1gnwJODgMc0vraqDhguTHAm8DHg13ZeGkjQtTh2XJC1PH2nPLx61MMnKwD50o0kfHSjfIMnbk3wvya/blNRftunot5nGOXgedJKHJTkuyVVJlk5Mpxx1jnaSVZO8KslJSX7epov+vk2F/fvJdizJOkmOGJjKenGSV7R/6KclyRpJ3prkwiTXJbm27fNe061jivp/keRnSe6V5P3t/Z+TnJ/k6W2dOUneluSnbT9+lmS/EXVNTE/+lyTbJflWkj+1x0lJthjTh3WSHDpQ/+/TTVN+whRtPDbJ19v6leSfkhSwAbDh0DTfwZ+d3dvPyU8Hjum5SV6Z5Db/9yT5dKvjb5K8PMlFrZ+/TnJUknuN2a+/SfKBgf36XZKzkxw4Zt0jklyWW6ZmfyXJlpN9fjOhqr4wHLJb+UXAF9rbnYYWv4EuZJ8B/PNwiG4hd2/gLLoR71cvQ3++CXwI2JRu9Hli1svhwNXAC6uqhjY7mC5kf7KqXjccslu9VVWnANtMty/NN9rzesMLkqzWfn8var9Xf0pyRpI9xlWW5B+TLGjr/jnJwiRvTnKPEevOS/L53DKl/6okP0zyvnR/O0nyC2DiZ27BwO/EcvvSUtL0OKItSVqePkH3z/Jzkry+qq4fWv4PdMHpm1V1+UD5zsCbgG8D5wPXARsDzwaelmTbFgSG/R1wNvAj4NPA6sA1k/RvPeC/6EbuvglcRTet9OnASUn2rapjR2y3Kt3U1DWB/27v9wQ+2PrwmknaBCDJvdv+bQ78kG5kcCXgycDnkjx81Ajb7bAqcCpwL+DL7f1zgC8l2RXYH9gCOAn4a9uPI5L8pqq+OKK+bYF/pQsgH6T7XHYHHp9k16r63sA+3gc4k24679l0U3vXo/scT03ykqoadZG47VsbZwAfA+4H/BR4B/A64Ca64DVhcLbEvwM30o3U/hJYG9gF+ACwJd0XO6P8J91056/RjajuArwU2LCV3yzJY+iO173pRli/BKwBPKL1+90D627V6rs33SjwxDHYDXhykqdV1TcG1p9D9zksqarZ/j/tr+15OLRNfHH2rhGhF4CqWpLkPcBXgJcAhy5Du2+iO51k/yRfp/vc1wL+aXhmSrrR9X9sb98xVcW3Y9bMru353KF2V6X7m7E93d+bD9L9DOwJHJfk36rqX4e2+XfgjXR/Zz5N93dtPnAI8KQkT66qv7Z1Hw18n27q+wnAYrrf4Y2BVwBvacveCzyTbsr9McDE8Vm6jPspqW9V5cOHDx8+fCy3B/B5uhHrvUcs+0pbtsdQ+f2BNUes/2i6f06/OlS+UaungHeO6cd3gZuGylYDNhix7jrAJXT/EK86tOwXrZ3vAPcYKL8v3TmhBWw7om8fHarn0638dUPl96T7B34psOk0j/F3W13bj+nrlwf3g+6LjAJ+TxdI1x5YtjFdwDpnqK5dB47xy4aWPauV/xjIQPnHWvkRQ+tvQvcFyA3A34xp40Vj9vUXwM8mORYbjihbCfhMq3fLMZ/D5cCDBspXofsCpoAtBspXpQszBTx7RFvDdVwG/HnEZ/MguusS/GLo52hOq/umcfs4os2J43YZcNCYx+FtnbHHbsTvwFV0YW7jgfKHtnr+wtDvxog61mzbF/CA6e5P23ar9nN4Q9v+82PWe8LE57cs9Y843kuHjtd76X6vltL9nVpzaLu3te1OAOYMlD8A+H9tu8cMlO/Q1l8M3G+o/a+3ZW8aKH9/K5s/os/34da/Z+9ixO+/Dx8+Zvcx6x3w4cOHDx937QfdyGAB3x0qX7/9I30lsMoy1Pd14Hpg5YGyiTD7y8HQMrTdbYL2FO28iaHQ3MonwuvjRmzzz23ZR0b07aMDZfdrAeT7Y9resm3znmn2daqg/ZAR20yExR1HLFtANyo8+M/8RJi7ZLB8RB+2a+9XpQuYVwPrjFj/4Lb+W0e0cc4k+zpp0J5ku22G22vlE0F77xHbvJihLxaAvVrZF6fR5sQXEAePWf76tvxJQ+WbAA9bhn0b/IJiqseUx47umglfauu/f2jZtq38F9Ps228Z+rJiGfZr4rP5E3DvMes8lxF/XwaW78ttv3TYbGD5RNAe91gE/OOIei+n+x3eaMSyl7Ztjx4oO6aV7Tti/YfTBfNLB8omgvYTpnGcDNo+fKyAj9mekiRJuus7DfhfYLs2HfqSVr4P3T+5x1abLjmonUP8UrrQuS63Pd3pPnQjboMuqKq/LEvnkmxKN51ze+CBdAFx0AYjNpuYmjzs9Pb86Cma3YZulDVJDhqxfKIPD5+inun4bVX9fET5FcDfcOtp1xN+CdyD7guBK4eWLaiqGrHN6cB2dPt+Jt006tWAs6rqjyPWPw04gNHH6uwRZdOS5L50n+dT6EZf1xhaZdTnCUNTg5v/157vPVD22PZ80jS687j2/NAxn/PD2vPDueVcYKrqx9Ooe5RvVdWuoxYk2Yhu+v10vJ9uavvpdMdyxqW7GNvEuc5r0Z1S8dnbUdW+dD+Xg34GDF9B/FZT9duU9EfSTXn/bPvb9fa27N50V2b/eVWNutXcae158Gd7i6FlN6uqS5L8Ctg4yZpVdS3wOeCVwFeTfIHu9I/vVdX/TrXDklYMBm1J0nJVVRMXqzqYbsT39UlCd7Xx4pYLpt0syeuB/6Cb2nwq8HO60dGiOx94U24biKG7vdC0Jdmu1b8S3VWLv0I3pXkp3T/GTxvTzlVjwuZE+2tP0fS67fkx7THOmlPUMx1Xjym/iS5cXDtmGXRTn4cNB+8Jw/s+8fyrMetPlK8zSV3LpJ0Tfi7wELoLcX2S7mfoJrovZl7F6M8TYNSXARPHYeWBson+/nIaXZr4nKe6uF0fn3MvkryX7jh9G3jqiC+uJj6b9ZKsWu2q4GPqWoNbvqSY7i2+aBcG+zTdZ/VKurD7oSTfqdveKmyiPw8cVVdV3Xxv6SSHAG+eTh/a78VZSXanm0XxliQfbu3fnp/t6WzzwLbetVX1/SQ7Am+lu6bBC9o+/Bg4qKo+P539kDR7DNqSpJlwDN09al+Q5C105yv+LXDa8IhQklWAt9P9Y75FVV05tHyHSdoZFX4n8za6Udcdquq7Q+28jS5oj7JekowI2w9oz+PCLUPLR96jeAV3/zHlw/t+9VD5sPWH1hu0rJ/jhJfQhey3VdW7Bhe0n5tX3c56B00E8nEj44Mm9m1+VX29h7aXm/bl1+F0wfZU4Ok14pZcVXVZG31dn+7e1N+cpNon0H2JdVlVLcuXJ++iu0jgEVX1oSRLgSPozvn/h6F1z6Y7BeWhSR5at76o4h1WVb9P8lNgM7oR6iu4fT/bg9uMmmFym22q6kxgfrvw2lZ0+/5KuhH2K6vq9GXbG0kzydt7SZKWuxaWT6C7YNgzueV+tEePWP3+dFNFvzsiZN+LqadlL4uNgN8Mh+zm8ZNstyq3TCEetFN7Pn+Kds+iC5OTfWmwotqhhbJhO7XniX2/hG6K/aPH3CJr5/Y87v7q4yzh1iPMgzZqz6Oulj7Z57ksJk4ZGA58k627Qn/O7fM8ii7EnQw8bVTIHjBxpfi3jvlZIN2t1N7a3o76PR/Xlx3pzl2/lDZtvaqOpLty+5OTvGxw/TbyPDG6e6urfPdoYlR+pdbmH+jC8oOT/O2I9Uf9bE/8Xuw0vHKSh9EF7Z+OmmFSVTdW1ZlV9S90dwkI8IyBVZa053G/F5JmgUFbkjRTJqaIv57u/M/fAsePWO9XdAFt6zb1FLh5OukHuPX5snfUYrrR6UcOFiZ5Kd1F3CZzyOC9b9u5wRP3sz1msg2r6ld052A+NslbJu6JO9SHjZI8ZOpdmHGb0J07f7Mkz6I7D/YndFfqpqpuoLv12dp0sxkG19+YLtT9hW6K8LL4HXC/Nso3bHF73mmova2Y5pThafgy3bnbuyfZc3hhkgcNvD2+9enVGXNf9iTbJlltqGyTFr6WuxaIP0Y3G+BrwDPbZzeZw+g+652AD4/o/+p0t6t7LHAht74V22R9uRfdLQGX0t3Ka/B2gPsCfwD+I8mGQ5u+hW4K+d5J/qO1P8pUp3SM6tMedNcyGL4uw8Tt+A7LwP3Zk9yPW/4OfHxofYC3JVl3YP05dLeWC93nMFF+m5+LZmJGyeCx+V17fvA0d0vSDHDquCRppnyDLnRs095/cNSFy6q7/+4HgDcAi5KcQDeC/AS6f5S/Q3+jk++jC9TfS/I/dFc33obuIlZfpLtq9Ci/oBt1v2igf3vQTQs9vAbuJT2J/ehGYN9DFxC+yy338X4E3VTRPRk9zXQ2nQQcnmQ+3RWZJ+6j/We6W3INTvueuMjca5JsQ/fZTdxHe01gvxq6L/I0fItuVsPJSRbQhfXzq+pE4Fi6L3I+0O4R/jO6+5o/le7znOpc6SlV1Y0tYJ8M/E+Sb9NNX74n3UXNdqQ7HWFi3d3buicnORO4gO5YPRjYmu6CbevR3cZqInhdQjdKORP/p72D7sKE19NdIOwtIwapz6uqEybeVNU1SZ5MN0vlxcBTk5xEF3YfQHdv6PvTjehONTo+6AN0Fxk7qKrOGVxQVVckeQXdlzefSLJjVS1ty36RZBe6K6W/Htg3ybforgy+tPVpO7rftyvpviQYttLQBevWAB4FTHxBckBVDV588VC6C7TtDlzY9n/iPtrr0d0x4OZgXlVntPPfXwdc3C5wdn07Vo+g+91470D9b6WbPbKg7cd1rT//QHfdgcFrW5xGN0Pm0CSb053esLSq3jNiPyXNlNm+7LkPHz58+Lj7POhGeiZumzP29kV0AeONdIHjz3Sj3J+kG1mauOXP4P2KR96reqjOkbf3Ap5ON5X7GroRs1PowuHErbr+aWj9X9AFuHWAI+nO2bwR+BHdKG2G1h/bN7qA/mrg+3TnZt5AF6xPBV4D3Geax3Wy23uNvJ3TuOPRlo06xhO3kPoXutByWjtm19AFyS3H1HVvuhHQn7XjNHGMdx2x7s1tTLKvawIfprsY2U3Dx5YujHyN7kuL6+gujrbvuM9h1L5Opz9054IfRffl0Y10MzR+QBfIhte9P10wu5guXF1LdwXw44Dncetb1d2R+2ifOsk6E/v/s6Hyif2f7DHy94ruyvQvofvy4yq6c6Wvojtv+0UM3F96Gvuwe2vrrMm2o5sJUmOO8z3ovjT4avv5uJHu78diupkI+wJrjPhbM2qf/0r3d+fLwC5j+nJPut+Hi+l+d6+huzXebe6vPrDN8+iuyj9xH/mL6EbkVx1a78l0XxxdQve34Vq6+9S/H3jwiHpfSDd7YOKikdP++fHhw8fyeaTq9l5vRJIk3V20EeJvMuJCY5Ik6dY8R1uSJEmSpB4ZtCVJkiRJ6pFBW5IkSZKkHnmOtiRJkiRJPfL2XhLwiU98ol74whfOdjckSZIkrbhuc//DcZw6LgHXXXfdbHdBkiRJ0l2EQVuSJEmSpB4ZtCVJkiRJ6pFBW5IkSZKkHhm0JUmSJEnqkUFbkiRJkqQeGbQlSZIkSeqRQVuSJEmSpB4ZtCVJkiRJ6pFBW5IkSZKkHhm0JUmSJEnqkUFbkiRJkqQeGbQlSZIkSeqRQVuSJEmSpB4ZtCVJkiRJ6pFBW5IkSZKkHhm0JUmSJEnqkUFbkiRJkqQeGbQlSZIkSeqRQVuSJEmSpB4ZtCVJkiRJ6pFBW5IkSZKkHhm0JUmSJEnqkUFbkiRJkqQeGbQlSZIkSeqRQVuSJEmSpB4ZtCVJkiRJ6pFBW5IkSZKkHhm0JUmSJEnq0ZzZ7oC0Ilj0y6uZe8CJs90NSZIkScDiQ+bPdhfuEEe0JUmSJEnqkUFbkiRJkqQeGbQlSZIkSeqRQVuSJEmSpB4ZtCVJkiRJ6pFBW5IkSZKkHhm0JUmSJEnqkUFbkiRJkqQeGbQlSZIkSeqRQVuSJEmSpB4ZtCVJkiRJ6pFBW5IkSZKkHhm0JUmSJEnqkUFbkiRJkqQeGbQlSZIkSeqRQVuSJEmSpB4ZtCVJkiRJ6pFBexJJrp2hdvZO8sAZauv0JFvNRFtD7c5N8tyB91slOXx51D1i+azssyRJkqS7J4P2imFvYEaC9vKUZM4ki+cCN4fhqjq3ql7dU9O3qluSJEmSZpNBexrSOSzJRUkWJdmrla+Z5FtJzmvlz2jlc5NckuQjSS5O8o0k9xxT9x7AVsBnklyQ5J5Jdklyfqvz40lWHbPtk5McN/B+pyRfa6+PTHJua/8dY7a/duD1HkmOba/XS/LFJOe0x3aTHJuDknwqyZnAp9q+L2jH5Lwk27ZVDwF2aPu4/1Bf75Pky0kWJvlBks0mae/xrY4L2jFaa0Td90zyufYZHA+MO/Yvacfo3CXXXz2uSUmSJElaJgbt6dkdmAdsDuwKHJZkfeAGYLeq2gLYGfjPJGnbbAx8qKoeCfwReNaoiqvqC8C5wPOqah5QwLHAXlW1KTAH2G9Mv04FHpNkjfZ+L+Bz7fWBVbUVsBnw+MnC6wjvB95XVVu3fn90ivUfAexaVc8BfgM8sR2TvYCJ6eEHAAuqal5VvW9o+3cA51fVZsBbgU9O0tYbgFe0Y7UD8OcRde8HXF9VDwfeDmw5qqKqOrqqtqqqrVZefe0pdlGSJEmSpsegPT3bA5+tqiVVdSXwHWBrIMB7kiykC70bAPdv21xeVRe01z+km948HQ9r217a3n8C2HHUilV1E3Ay8LQ2bXs+8JW2+NlJzgPOBx5JF4ana1fgg0kuAE4A7pVkzUnWP6Gq/txerwJ8JMki4Lhptrs98Km2T6cB6ya515h1zwTem+TVwDrtGAzbEfh0q28hsHAafZAkSZKkXkx2Tq2m9jxgPWDLqvprksXAam3ZjQPrLWHM9OUefA54JfB74NyquibJQ+lGfreuqj+0KeGrjdi2Bl4PLl8JeGxV3TDNPlw38Hp/4Eq60f+V6Eb9e1NVhyQ5EXgKcGaSv++zfkmSJEm6oxzRnp4FwF5JVk6yHt2I6dnA2sBvWsjeGXjI7az/GmCt9vonwNwkG7X3z6cbQR/nO8AWwIu5Zdr4vejC79VJ7g/8w5htr0zy8CQrAbsNlH8DeNXEmyTzlmFf1gZ+VVVLW99XbuWD+zhsAd2XFiTZCfhtVf1p1IpJNqyqRVV1KHAOsMmIus+gXRwtyaPops9LkiRJ0owwaE/P8XTTjy8ETgPeVFW/Bj4DbNWmSb8A+PHtrP9Y4Kg2VTvAPsBxrd6lwFHjNqyqJcDX6ML011rZhXRTxn8M/DfddOtRDmjbfA/41UD5q9t+LUzyI+Bly7AvRwAvTHIhXQieGO1eCCxJcmGS/Ye2OQjYsk3BPwR44ST1v7ZdlG4h8FfgpBF1HwmsmeQS4J10U/clSZIkaUakqqZeS7qL2+/Ag+ukJQ58S5IkSSuCxYfMn+0ujJKpV+k4oi1JkiRJUo+8GNoMSvIhYPie1O+vqmOmse3xwEOHit9cVaf01b8p2t8HeM1Q8ZlV9Yq7QnuSJEmS1BeD9gy6IyGxqnabeq3lp30ZMOUXAnfW9iRJkiSpL04dlyRJkiSpRwZtSZIkSZJ6ZNCWJEmSJKlHBm1JkiRJknpk0JYkSZIkqUcGbUmSJEmSemTQliRJkiSpRwZtSZIkSZJ6ZNCWJEmSJKlHc2a7A9KKYNMN1ubIl8+f7W5IkiRJugtwRFuSJEmSpB4ZtCVJkiRJ6pFBW5IkSZKkHhm0JUmSJEnqkUFbkiRJkqQeGbQlSZIkSeqRQVuSJEmSpB4ZtCVJkiRJ6pFBW5IkSZKkHs2Z7Q5IK4JFv7yauQecONvdkO7UFh8yf7a7IEmStEJwRFuSJEmSpB4ZtCVJkiRJ6pFBW5IkSZKkHhm0JUmSJEnqkUFbkiRJkqQeGbQlSZIkSeqRQVuSJEmSpB4ZtCVJkiRJ6pFBW5IkSZKkHhm0JUmSJEnqkUFbkiRJkqQeGbQlSZIkSeqRQVuSJEmSpB4ZtCVJkiRJ6pFBW5IkSZKkHhm0JUmSJEnqkUFbkiRJkqQeGbTvhpLMTfLcGWpr7yQfnIF2Xptk9dnsgyRJkiSBQXuFlWTl5VTvHGAuMCNBewa9FhgZtCVJkiRpJhm0Z0EbUf5xks8kuSTJF5KsnmRxkkOTnAfsmWRekh8kWZjk+CT3btufnuT9SS5IclGSbVr5Gkk+nuTsJOcneUYr3zvJCUlOA74FHALs0LbfP8kZSeYN9O+7STYf0e+VWh/XGSj7aZL7J3lakrNau6cmuf+I7Y9NssfA+2sHXr8xyTltX98xybFbI8mJSS5s+75XklcDDwS+neTbbb19klya5GxguzF1vSTJuUnOXXL91eOalCRJkqRlYtCePQ8DjqiqhwN/Al7eyn9XVVtU1eeATwJvrqrNgEXA2we2X72q5rXtPt7KDgROq6ptgJ2Bw5Ks0ZZtAexRVY8HDgAWVNW8qnof8DFgb4AkfwesVlUXDne4qpYCXwF2a+s+Bvh5VV0JfBd4bFU9Gvgc8KbpHogkTwI2BrYB5gFbJtlxzOpPBq6oqs2r6lHAyVV1OHAFsHNV7ZxkfeAddAF7e+ARoyqqqqOraquq2mrl1deebnclSZIkaVIG7dnz/6rqzPb603SBEODzAEnWBtapqu+08k8Ag+HzswBVdQZwrzbK/CTggCQXAKcDqwEPbut/s6p+P6YvxwFPTbIKsC9w7CT9/jywV3v9jxP9BR4EnJJkEfBG4JGT1DHsSe1xPnAesAld8B5lEfDENvK/Q1WNGop+DHB6VV1VVX8Z6KMkSZIkLXcG7dlTY95fdwe2D/CsNlI9r6oeXFWXTFVvVV0PfBN4BvBs4DOTtPt9YKMk6wHPBL7Uyj8AfLCqNgVeShfyh91E+5lLshJwj1Ye4OCBfm9UVR8b09dL6UbnFwHvSvKvk/RVkiRJkmacQXv2PDjJ49rr59JNvb5ZG6n9Q5IdWtHzge8MrLIXQJLtgavb+qcAr0qStuzRY9q+BlhrqOyjwOHAOVX1h3GdrqoCjgfeC1xSVb9ri9YGftlev3DM5ouBLdvrpwOrtNenAPsmWbP1e4Mk9xtVQZIHAtdX1aeBw+hC9/A+nQU8Psm6bZR+z3H7I0mSJEl9mzPbHbgb+wnwiiQfB34EHAm8amidFwJHtdtWXQbsM7DshiTn04XVfVvZvwH/BSxsI8aXA08d0fZCYEmSC4Fjq+p9VfXDJH8CjplG3z8PnEM7r7s5CDguyR+A04CHjtjuI8BXWrsn00bZq+obSR4OfL99R3At8E/Ab0bUsSnduedLgb8C+7Xyo4GTk1zRztM+iG70/Y/ABdPYJ0mSJEnqRboBSs2kJHOBr7WLed2e7U8H3lBV5/bYpwfSnde9Sbvo2d3KfgceXCct2Wy2uyHdqS0+ZP5sd0GSJGl5ynRXdOq4SPICuunWB94dQ7YkSZIk9cmp47OgqhYDt2s0u22/U2+d6er7JN2txG6WZB/gNUOrnllVr+iz7XGSrEt3z+9huwycFy5JkiRJKxyDtkaqqmOY3vnay6v939HdU1uSJEmS7lScOi5JkiRJUo8M2pIkSZIk9cigLUmSJElSjwzakiRJkiT1yKAtSZIkSVKPDNqSJEmSJPXIoC1JkiRJUo8M2pIkSZIk9cigLUmSJElSjwzakiRJkiT1aM5sd0BaEWy6wdoc+fL5s90NSZIkSXcBjmhLkiRJktQjg7YkSZIkST0yaEuSJEmS1CODtiRJkiRJPTJoS5IkSZLUI4O2JEmSJEk9MmhLkiRJktQjg7YkSZIkST0yaEuSJEmS1KM5s90BaUWw6JdXM/eAE2e7G7obWnzI/NnugiRJknrmiLYkSZIkST0yaEuSJEmS1CODtiRJkiRJPTJoS5IkSZLUI4O2JEmSJEk9MmhLkiRJktQjg7YkSZIkST0yaEuSJEmS1CODtiRJkiRJPTJoS5IkSZLUI4O2JEmSJEk9MmhLkiRJktQjg7YkSZIkST0yaEuSJEmS1CODtiRJkiRJPTJoS5IkSZLUI4P2CizJtbPdh5mUZG6S596O7dZJ8vJJlh+bZI871jtJkiRJmh6DtlYkc4FlDtrAOsDYoC1JkiRJM8mgfSeQzmFJLkqyKMlerXzNJN9Kcl4rf0YOusynAAAgAElEQVQrn5vkkiQfSXJxkm8kueck9b84yTlJLkzyxSSrt/JjkxyZ5AdJLkuyU5KPt7qPHdj+Oa39i5IcOlB+7cDrPSa2afUenuR7rd6J0eZDgB2SXJBk/zF9fWSSs9s6C5Ns3LbbsJUd1o7XB5P8JMmpwP3G1PWSJOcmOXfJ9VdP45OQJEmSpKkZtO8cdgfmAZsDuwKHJVkfuAHYraq2AHYG/jNJ2jYbAx+qqkcCfwSeNUn9X6qqratqc+AS4EUDy+4NPA7YHzgBeB/wSGDTJPOSPBA4FHhC6+PWSZ45jX1aH9geeCpdUAY4AFhQVfOq6n1jtnsZ8P6qmgdsBfyibfe/bbs3ArsBDwMeAbwA2HZURVV1dFVtVVVbrbz62tPosiRJkiRNzaB957A98NmqWlJVVwLfAbYGArwnyULgVGAD4P5tm8ur6oL2+od007LHeVSSBUkWAc+jC9ITvlpVBSwCrqyqRVW1FLi41bk1cHpVXVVVNwGfAXacxj59uaqWVtWPBvo8Hd8H3prkzcBDqurPI9bZkVuO1xXAactQvyRJkiTdIQbtO7fnAesBW7YR3iuB1dqyGwfWWwLMmaSeY4FXVtWmwDsG6hisZ+lQnUunqBOgBl6vNrRssK4wTVX138DTgT8DX0/yhOluK0mSJEkzwaB957AA2CvJyknWoxuxPRtYG/hNVf01yc7AQ25n/WsBv0qyCl14XxZnA49Pct8kKwPPoRtxB7gyycOTrEQ3nXsq17S+jJXkb4HLqupw4CvAZiO2O4Nbjtf6dNPqJUmSJGlGTDUiqRXD8XTnSV9IN0r8pqr6dZLPAF9tU77PBX58O+t/G3AWcFV7njTsDqqqXyU5APg23cj0iVX1lbb4AOBrrd5zgTWnqG4hsCTJhcCxY87Tfjbw/CR/BX4NvKeqfp/kzCQXAScBb6I7Z/xHwP/RTTeXJEmSpBmR7vRb6e5tvwMPrpOWbDbb3dDd0OJD5s92FyRJkjQ90z7l1anjkiRJkiT1yKnjdyNJPgRsN1T8/qo6Zjb6M5kkf09327BBl1fVdM71liRJkqRZY9C+G6mqV8x2H6arqk4BTpntfkiSJEnSsnLquCRJkiRJPTJoS5IkSZLUI4O2JEmSJEk9MmhLkiRJktQjg7YkSZIkST0yaEuSJEmS1CODtiRJkiRJPTJoS5IkSZLUI4O2JEmSJEk9MmhLkiRJktSjObPdAWlFsOkGa3Pky+fPdjckSZIk3QU4oi1JkiRJUo8M2pIkSZIk9cigLUmSJElSjwzakiRJkiT1yKAtSZIkSVKPDNqSJEmSJPXIoC1JkiRJUo8M2pIkSZIk9cigLUmSJElSj+bMdgekFcGiX17N3ANOnO1uaIYsPmT+bHdBkiRJd2GOaEuSJEmS1CODtiRJkiRJPTJoS5IkSZLUI4O2JEmSJEk9MmhLkiRJktQjg7YkSZIkST0yaEuSJEmS1CODtiRJkiRJPTJoS5IkSZLUI4O2JEmSJEk9MmhLkiRJktQjg7YkSZIkST0yaEuSJEmS1CODtiRJkiRJPTJoS5IkSZLUI4O2JEmSJEk9MmhLkiRJktSju33QTnJQkjdMsvzYJHuMKJ+X5ClT1L1Tkm1vZ79eluQFt2fbFVGS1yZZfeD915OsszzqHlq2d5IP9tGOJEmSJE3H3T5o3wHzgEmDNrATcLuCdlUdVVWfvD3bzoZ0Jvt5ei1wcxiuqqdU1R97av5WdUuSJEnSbLpbBu0kBya5NMl3gYe1sg2TnJzkh0kWJNlkYJNdk5zbtnlqknsA7wT2SnJBkr1GtDEXeBmwf1tnhyRzk5yWZGGSbyV58CR9vHmkPcnpSQ5Ncnbrww6tfOUk/5Hkolbnq1r5LknOT7IoyceTrNrKFyc5uPXn3CRbJDklyf8medlA229Mck6r8x2T9HFukp8k+SRwEfA3SY5sdV88sW2SVwMPBL6d5NsDfblve/26tg8XJXntJO2tkeTEJBe2dfcaU/c+7TidDWw3SX0vaX09d8n1V49bTZIkSZKWyd0uaCfZEvhHbhmR3rotOhp4VVVtCbwBOGJgs7nANsB84Ci64/avwOeral5VfX64napa3NZ9X1tnAfAB4BNVtRnwGeDwZej6nKrahm709u2t7CWtb/Mm6kyyGnAssFdVbQrMAfYbqOf/qmoesKCttwfwWGAiFD8J2Ljt7zxgyyQ7TtKvjYEjquqRVfVz4MCq2grYDHh8ks2q6nDgCmDnqtp5cOP2eewDPKb148VJHj2mrScDV1TV5lX1KODk4bqTrN/2ZTtge+AR4zpeVUdX1VZVtdXKq689yS5KkiRJ0vTd7YI2sANwfFVdX1V/Ak4AVqOb4n1ckguADwPrD2zzP1W1tKp+ClwGbDJc6TQ9Dvjv9vpTdEFwur7Unn9IF64BdgU+XFU3AVTV7+lG6C+vqkvbOp8ABoPyCe15EXBWVV1TVVcBN7Zzpp/UHucD59Ht68aT9OvnVfWDgffPTnJe2/6RTBJ0m+3pPo/rquratp87jFl3EfDENrq/Q1WNGoZ+DHB6VV1VVX8BbvMliCRJkiQtT3NmuwMriJWAP7aR3lFqivcz4cb2vIQ79rlN1LN04PXE+zlAgIOr6sPTrO+6iRdJHko3G2DrqvpDkmPpvsToRVVdmmQLupkI70ryrap6Z1/1S5IkSVIf7o4j2mcAz0xyzyRrAU8DrgcuT7In3Hxhr80HttkzyUpJNgT+FvgJcA2w1hRtDa/zPbpp6wDPo5u+fUd8E3hpkjmt3/dpfZubZKO2zvOB7yxDnacA+yZZs9W5QZL7TXPbe9EF76uT3B/4h4Fl447XArrPY/UkawC7Mea4JHkgcH1VfRo4DNhiRN1n0U1ZXzfJKsCe0+y7JEmSJPXibhe0q+o8uunEFwInAee0Rc8DXpTkQuBi4BkDm/0fcHZb/2VVdQPwbeAR4y6G1nwV2G3iYmjAq4B9kiykC8CvuYO789HWt4Wt389tfduHbhr8IrqR6qOmW2FVfYNuevv32/ZfYOovFCa2vZBuyviPWx1nDiw+Gjh54oJlA9ucR3eu+Nl0IfmjVXX+mCY2Bc5u0/vfDrxruO6q+hVwEPD91v4l0+m7JEmSJPUlVbMxC1pasex34MF10pLNZrsbmiGLD5k/212QJEnSnU+mu+LdbkRbkiRJkqTlyYuh9SDJPtx2GviZVfWKaWx7ILc9j/i4qnp3X/27o5KsC3xrxKJdqup3d/b2JEmSJKlPBu0eVNUxwDG3c9t3AytMqB6lhdtxV2S/07cnSZIkSX1y6rgkSZIkST0yaEuSJEmS1CODtiRJkiRJPTJoS5IkSZLUI4O2JEmSJEk9MmhLkiRJktQjg7YkSZIkST0yaEuSJEmS1CODtiRJkiRJPTJoS5IkSZLUozmz3QFpRbDpBmtz5Mvnz3Y3JEmSJN0FOKItSZIkSVKPDNqSJEmSJPXIoC1JkiRJUo8M2pIkSZIk9cigLUmSJElSjwzakiRJkiT1yKAtSZIkSVKPDNqSJEmSJPXIoC1JkiRJUo/mzHYHpBXBol9ezdwDTpztbqwQFh8yf7a7IEmSJN2pOaItSZIkSVKPDNqSJEmSJPXIoC1JkiRJUo8M2pIkSZIk9cigLUmSJElSjwzakiRJkiT1yKAtSZIkSVKPDNqSJEmSJPXIoC1JkiRJUo8M2pIkSZIk9cigLUmSJElSjwzakiRJkiT1yKAtSZIkSVKPDNqSJEmSJPXIoC1JkiRJUo8M2pIkSZIk9cig3ZMk35vGOq9NsvpM9GegzWcmecRMtjnU/pTHZWj9Y5PssYzbjN3HJHOTXLQs9UmSJEnSHWHQ7klVbTuN1V4LjAzaSVa+vW1Pse0zgRkP2knmwLSPyx01K/soSZIkSaMYtHuS5Nr2vFOS05N8IcmPk3wmnVcDDwS+neTbE9sk+c8kFwKPS7JLkvOTLEry8SSrTtLe4iSHJjkP2DPJi5Ock+TCJF9MsnqSbYGnA4cluSDJhu1xcpIfJlmQZJNJ2jg2yVFJzk1yaZKntvKVkxzW2luY5KUD+74gyQnAj4aOS9o2F7X922ug/INJfpLkVOB+UxznQ5L8qLX7H2P2cct2HC4EXjFJXS9p+3bukuuvnqxZSZIkSZq2ObPdgbuoRwOPBK4AzgS2q6rDk7wO2LmqftvWWwM4q6pen2Q14KfALlV1aZJPAvsB/zVJO7+rqi0AkqxbVR9pr98FvKiqPtBC79eq6gtt2beAl1XVT5M8BjgCeMIkbcwFtgE2pPuSYCPgBcDVVbV1+zLgzCTfaOtvATyqqi4fqmd3YB6wOXBf4JwkZwCPAx5GNyJ9f7qA/vFRHUmyLrAbsElVVZJ1quqPI/ZxIfDKqjojyWHjdqyqjgaOBtjvwIOLJZMcBUmSJEmaJke0l4+zq+oXVbUUuIAurI6yBPhie/0w4PKqurS9/wSw4xTtfH7g9aPaaPIi4Hl0Qf9WkqwJbAscl+QC4MPA+lO08T9VtbSqfgpcBmwCPAl4QavjLGBdYOO2/tkjQjbA9sBnq2pJVV0JfAfYuu3jRPkVwGmT9OVq4AbgY0l2B64fsY/rAOtU1Rmt6FNT7J8kSZIk9coR7eXjxoHXSxh/nG+oqjsyjnrdwOtjgWdW1YVJ9gZ2GrH+SsAfq2reMrRRI94HeFVVnTK4IMlOQ33qVVXdlGQbYBdgD+CVTD4aL0mSJEkzzhHtmXUNsNaYZT8B5rap2QDPpxv1na61gF8lWYVuRPs2bVbVn4DLk+wJN58fvfkU9e6ZZKUkGwJ/2/p5CrBfa4skf5dkjSnqWQDs1c7vXo9uJPts4IyB8vWBncdV0Ebk166qrwP7001DH97HPwJ/TLJ9W/a821QkSZIkScuRQXtmHQ2cPHExtEFVdQOwD9207kXAUuCoZaj7bXTTuM8EfjxQ/jngje0iaxvSBc8XtQuFXQw8Y4p6/48uEJ9Ed273DcBH6c6lPi/drbM+zNSzI44HFgIX0k0Pf1NV/bqV/7TV90ng+5PUsRbwtXYO9neB143Zx32AD7Wp7ZmiX5IkSZLUq1QNzwyWOkmOZeAiY3dl+x14cJ20ZLPZ7sYKYfEh82e7C5IkSdKKaNqDeI5oS5IkSZLUIy+GtoJLcjzw0KHiNw9fiOwOtnEgsOdQ8XFVtXdfbSyrmdhvSZIkSVoeDNoruKrabQbaeDfw7uXdzrKYif2WJEmSpOXBqeOSJEmSJPXIoC1JkiRJUo8M2pIkSZIk9cigLUmSJElSjwzakiRJkiT1yKAtSZIkSVKPDNqSJEmSJPXIoC1JkiRJUo8M2pIkSZIk9cigLUmSJElSj+bMdgekFcGmG6zNkS+fP9vdkCRJknQX4Ii2JEmSJEk9MmhLkiRJktQjg7YkSZIkST0yaEuSJEmS1CODtiRJkiRJPTJoS5IkSZLUI4O2JEmSJEk9MmhLkiRJktQjg7YkSZIkST2aM9sdkFYEi355NXMPOHG2uzHW4kPmz3YXJEmSJE2TI9qSJEmSJPXIoC1JkiRJUo8M2pIkSZIk9cigLUmSJElSjwzakiRJkiT1yKAtSZIkSVKPDNqSJEmSJPXIoC1JkiRJUo8M2pIkSZIk9cigLUmSJElSjwzakiRJkiT1yKAtSZIkSVKPDNqSJEmSJPXIoC1JkiRJUo8M2pIkSZIk9cigLUmSJElSjwzakiRJkiT1yKDdkyTvTLLrFOvslGTbmepTa3NukufOZJtD7U95XIbW3ynJ15axjUn3McnpSbZaljolSZIk6fYyaPekqv61qk6dYrWdgJFBO8mc29v2FNvOBWYlaCdZeZrH5Y6ayyztoyRJkiQNM2hz84joJUk+kuTiJN9Ics8k85L8IMnCJMcnufckdRybZI/2enGSdyQ5L8miJJskmQu8DNg/yQVJdmjbHJXkLODfk9wnyZdbez9Istkk7R2U5FNJzgQ+1fZhQWvzvIGR80OAHVqb+ydZOclhSc5p7bx0kjZ2SnJGkhOT/KT1daW27ElJvt/aOi7JmgP7fmiS84A9h47LLknOb8fk40lWbeVPTvLjts3uU3xWj2/7ckGra60R+3jPJJ9rn+nxwD3H1PWSJOcmOXfJ9VdP1qwkSZIkTZtB+xYbAx+qqkcCfwSeBXwSeHNVbQYsAt6+DPX9tqq2AI4E3lBVi4GjgPdV1byqWtDWexCwbVW9DngHcH5r762t/ck8Ati1qp4D/AZ4YmtzL+Dwts4BwILW5vuAFwFXV9XWwNbAi5M8dJI2tgFe1draENg9yX2Bf2ltbwGcC7xuYJvfVdUWVfW5iYIkqwHHAntV1abAHGC/Vv4R4GnAlsADptjnNwCvqKp5wA7An0fs437A9VX1cLrPbMtRFVXV0VW1VVVttfLqa0/RrCRJkiRNj0H7FpdX1QXt9Q/pQuU6VfWdVvYJYMdlqO9LA3XNnWS946pqSXu9PfApgKo6DVg3yb0m2faEqvpze70K8JEki4Dj6ILxKE8CXpDkAuAsYF26LxnGObuqLmt9/Gzr42Nb/We2el4IPGRgm8+PqOdhdMf40vZ+4nhu0sp/WlUFfHqSvgCcCbw3yavpPp+bRqyz40Q9VbUQWDhFnZIkSZLUm9t9XvBd0I0Dr5cA6/RU3xImP87X3YE2BrfdH7gS2JzuC5QbxmwT4FVVdco026gR7wN8s42kT9WvXlXVIUlOBJ5CF/T/fnm1JUmSJEm3hyPa410N/CHJDu3984HvTLL+dFwDrDXJ8gXA86A7P5pu+vmfpln32sCvqmopXV9XHtPmKXRTtldp7fxdkjUmqXebJA9t52bvBXwX+AGwXZKNWh1rJPm7Kfr3E2DuxDbccjx/3Mo3bOXjwjutrQ2ralFVHQqcQzciPryPZ9AujpbkUcDYc90lSZIkqW+OaE/uhcBRSVYHLgP2uYP1fRX4QpJn0J33POwg4ONJFgLXt/an6wjgi0leAJzMLaPKC4ElSS6kO0f6/XRT2c9LEuAq4JmT1HsO8EFgI+DbwPFVtTTJ3sBnJy5oRnfO9qWjq4CquiHJPsBx7Srp5wBHVdWNSV4CnJjkerovGyb7MuK1SXYGlgIXAye114P7eCRwTJJLgEvopu9LkiRJ0oxId1qsdFttVP0NVfXU2e7L8rbfgQfXSUtW3IHvxYfMn+0uSJIkSXd3me6KTh2XJEmSJKlHTh1fRkk+BGw3VPz+qjpmObW3D/CaoeIzq+oVPbaxKe1q5wNurKrHAKf31c6ymIn9liRJkqTlwaC9jGY66LUAv1xC/EAbi4B5y7ONZTUT+y1JkiRJy4NTxyVJkiRJ6pFBW5IkSZKkHhm0JUmSJEnqkUFbkiRJkqQeGbQlSZIkSeqRQVuSJEmSpB4ZtCVJkiRJ6pFBW5IkSZKkHhm0JUmSJEnq0ZzZ7oC0Ith0g7U58uXzZ7sbkiRJku4CHNGWJEmSJKlHBm1JkiRJknpk0JYkSZIkqUcGbUmSJEmSemTQliRJkiSpRwZtSZIkSZJ6ZNCWJEmSJKlHBm1JkiRJknpk0JYkSZIk6f+3d//RvpV1ncDfH7kqoxCmTC7HH11FzAGBS1wxTc3S8RcNZIloNiY5EmKM5OiKhpaVyYA61YiJyliChg3CsqIUzEEMAkEhhAsmavwwrCYtZUAMFT7zx3dfOl3PPfecex/Ouefe12uts87+7v3svT/f/X3Wvff9fZ6970BrVroA2B5s+PKtWXv8hxdsc9PJhyxTNQAAwGpmRBsAAAAGErQBAABgIEEbAAAABhK0AQAAYCBBGwAAAAYStAEAAGAgQRsAAAAGErQBAABgIEEbAAAABhK0AQAAYCBBGwAAAAYStAEAAGAgQRsAAAAGErQBAABgIEEbAAAABhK0AQAAYCBBGwAAAAYStLlHVV260jVsjapaV1XPX2D7TVW153LWBAAA7LwEbe7R3U9Z6Rq20rokmw3aAAAAy0nQ5h5Vdfv0+xlV9YmqOqeqPldVZ1ZVTdueWFWXVtXVVfWpqtq9qnatqvdW1YaquqqqfnRq+/Kq+qOq+tg0qvwLVfXaqc1lVfXgqd1eVXV+VV1ZVRdX1eMXqPHwqrp2Ov9FVXW/JG9MckRVfaaqjqiqh1TVn1XVdVX1niS1mWMdVVVXVNUVd91x6+CrCQAA7KwEbTbnwCTHJdknyWOS/PAUas9K8pruPiDJs5J8M8mrk3R375fkJUnOqKpdp+M8IclPJnlikhOT3NHdByb5ZJKXTW1OS3Jsdx+U5HVJTl2grjckec50/kO7+1vTurO6e113n5XkV5P8RXfvm+QPkzxqvgN192ndvb671+/ygD2Wen0AAADmtWalC2C79anuviVJquozSdYmuTXJ33X3p5Oku//ftP2pSd4+rftcVd2c5HHTcS7s7tuS3FZVtyb5k2n9hiT7V9VuSZ6S5Oxp0DxJ7r9AXZckOb2qPpjkQ5tp8/TMwn26+8NV9bWlvHEAAIBtIWizOXfOWb4rW99X5h7n7jmv756OeZ8kX+/udYs5WHcfXVVPSnJIkiur6qCtrAsAAOBeYeo4S3F9kodV1ROTZLo/e02Si5O8dFr3uMymal+/mANOo+I3VtXh0/5VVQdsrn1V7dXdl3f3G5J8Jckjk9yWZPc5zS5K8tNT++cl+d4lvUsAAIBtIGizaNP90EckeXtVXZ3kY0l2zeye6vtU1YbM7uF+eXffufkjfZeXJnnFdMzrkhy2QNu3Tg9duzbJpUmuTnJhkn02Pgwtya8neXpVXZfZFPIvLemNAgAAbIPq7pWuAVbcq044qc+7a/8F29x08iHLVA0AALAdmvd/M5qPEW0AAAAYyMPQ2C5V1QlJDt9k9dndfeJK1AMAALBYgjbbpSlQC9UAAMCqY+o4AAAADCRoAwAAwECCNgAAAAwkaAMAAMBAgjYAAAAMJGgDAADAQII2AAAADCRoAwAAwECCNgAAAAwkaAMAAMBAa1a6ANge7PfwPfLOYw5Z6TIAAIAdgBFtAAAAGEjQBgAAgIEEbQAAABhI0AYAAICBBG0AAAAYSNAGAACAgQRtAAAAGEjQBgAAgIEEbQAAABhozUoXANuDDV++NWuP//A9r286+ZAVrAYAAFjNjGgDAADAQII2AAAADCRoAwAAwECCNgAAAAwkaAMAAMBAgjYAAAAMJGgDAADAQII2AAAADCRoAwAAwECCNgAAAAwkaAMAAMBAgjYAAAAMJGgDAADAQII2AAAADCRoAwAAwECCNgAAAAwkaDNcVb21qq6rqrfeC8d+T1XtM/q4AAAAo6xZ6QLYIR2V5MHdfddiGlfVmu7+zmLadvd/3qbKAAAA7mVGtPlXquplVXVNVV1dVe+vqrVV9fFp3QVV9aip3elVdUpVXVpVN1TVC6f15ybZLcmVVXXEFvZ/V1VdnuQtVfVrVXVGVV1cVTdX1U9W1VuqakNVnV9V9532+0RVrZ+Wb6+qE6daL6uqh07r95peb6iqN1XV7StwKQEAgJ2UoM09qmrfJL+S5Me6+4Akr0ny9iRndPf+Sc5McsqcXR6W5KlJfjzJyUnS3Ycm+WZ3r+vus7aw/yOSPKW7Xzu93ivJjyU5NMnvJ7mwu/dL8s0kh8xT8gOTXDbVelGSV07r35bkbdO+tyzwfo+qqiuq6oq77rh1yxcIAABgEQRt5vqxJGd391eTpLv/KcmTk3xg2v7+zIL1Rn/U3Xd392eTPHQzx1xo/7M3mV5+Xnd/O8mGJLskOX9avyHJ2nmO/a0kfzotXzmnzZOTnD0tfyCb0d2ndff67l6/ywP22FwzAACAJRG02RZ3zlmurdj/G/Mdr7vvTvLt7u5p/d2Z/3kCc9vctZk2AAAAy0rQZq6PJzm8qh6SJFX14CSXJnnxtP2lSS5e4jG3df+tcVmSn5qWX7xQQwAAgNGMAHKP7r6uqk5M8udVdVeSq5Icm+S9VfX6JF9JcuQSD7ut+2+N45L8flWdkNn0czdgAwAAy6b+ZeYt7Biq6gGZPZCtq+rFSV7S3YcttM+rTjipz7tr/3te33TyfM9eAwAAdmKLvl3WiDY7ooOS/E5VVZKvJ/m5Fa4HAADYiQja7HC6++IkB6x0HQAAwM7Jw9AAAABgIEEbAAAABhK0AQAAYCBBGwAAAAYStAEAAGAgQRsAAAAGErQBAABgIEEbAAAABhK0AQAAYCBBGwAAAAYStAEAAGCgNStdAGwP9nv4HnnnMYesdBkAAMAOwIg2AAAADCRoAwAAwECCNgAAAAwkaAMAAMBAgjYAAAAMJGgDAADAQII2AAAADCRoAwAAwECCNgAAAAwkaEOSDV++daVLAAAAdhCCNgAAAAwkaAMAAMBAgjYAAAAMJGgDAADAQII2AAAADCRoAwAAwECCNgAAAAwkaAMAAMBAgjYAAAAMJGgDAADAQII2AAAADCRoAwAAwECCNgAAAAwkaAMAAMBAgjYAAAAMJGgDAADAQII2AAAADLTkoF1Vv1ZVr1tg++lV9cJ51q+rqudv4djPqKqnLLWmad+jq+plW7Pv9qiqjquqB8x5/ZGqetC9dK5Dq+r4rdjv0kHnX1tV36yqz8z5ud8C7Z9RVX864twAAACjrVnGc61Lsj7JRxZo84wktydZcoDr7ndtXVkro6oqSXX33ZtpclyS309yR5J094JfUmyL7j43yblbsd9WfSmyGX/d3esGHg8AAGBFLGpEu6pOqKrPV9VfJPmBad1eVXV+VV1ZVRdX1ePn7PKsqrpi2ufHp9HJNyY5YhqtPGKec6xNcnSSX5zaPG0a6fx4VV1TVRdU1aMWqPGekfaq+kRVvbmqPjXV8LRp/S5V9T+q6trpmMdO659ZVVdV1Yaq+r2quv+0/qaqOmmq54qq+sGq+mhV/XVVHT3n3K+vqk9Px/z1BWpcW1XXV9X7klyb5JFV9c7p2Ndt3Leq/kuSf5fkwqq6cE4te07Lr53ew7VVddwWzve5adz1WloAAA8GSURBVJbB56vqzKp6VlVdUlVfqKqDp3Yvr6rfmZYPn457dVVdNK3bd7qWn5ne497T+tun38+Yrvk50/nOnL5ISFU9f1p3ZVWdspSR6Ko6uKo+OX02l1bVD8zT5kfmjIJfVVW7L/Yzqaqjpmt/xV133LrYsgAAABa0xaBdVQcleXFmI9LPT/LEadNpSY7t7oOSvC7JqXN2W5vk4CSHJHnXdJ43JDmru9d191mbnqe7b5ra/vbU5uIkb09yRnfvn+TMJKcs4b2t6e6DMxsZ/tVp3VFTbes2HrOqdk1yepIjunu/zEb5XzXnOF+aRlovntq9MMkPJdkYip+dZO/p/a5LclBVPX2BuvZOcmp379vdNyc5obvXJ9k/yY9U1f7dfUqSv03yo939o3N3nj6PI5M8aarjlVV14ALne2yS30zy+Onnp5M8NbPP7L/N0/4NSZ7T3QckOXRad3SSt03XYX2SW+bZ78DMrvU+SR6T5Iena/vuJM+b+sm/XaDOveYE5ndM6z6X5GndfeBU13+fZ7/XJXn1VNvTknxzsZ9Jd5/W3eu7e/0uD9hjgdIAAAAWbzFTx5+W5A+7+44kqapzk+ya5ClJzp4GLpPk/nP2+eA0JfoLVXVDZgFvazw5yU9Oy+9P8pYl7Puh6feVmYXrJHlWknd193eSpLv/qaoOSHJjd39+anNGklcn+Z/T641Tqjck2a27b0tyW1XdWbN7pp89/Vw1tdsts5B30Wbqurm7L5vz+kVVdVRmn8XDMguq1yzwvp6a2efxjSSpqg9l9hldtZn2N3b3hqntdUku6O6uqg1zrstclyQ5vao+mH+5hp9MckJVPSLJh7r7C/Ps96nuvmU6z2emY9+e5IbuvnFq8weZfdkxn/mmju+R5IxpBL2T3Hcz9f5WVZ051XbLFLSX8pkAAAAMs7X3aN8nydcXuKe2t/B6Odw5/b4r23Yv+sbj3D1neePrNUkqyUnd/e5FHu8bGxeq6tGZjcg+sbu/VlWnZ/Ylxkib1jz3/XzXdenuo6vqSZnNRriyqg7q7g9U1eXTuo9U1c9398cXOM+2XvONfiPJhd39gprdWvCJeeo9uao+nNlsi0uq6jlZ+mcCAAAwzGLu0b4oyU9U1b+Z7n/9j5k9oOvGqjo8mT3YaxoZ3ujwqrpPVe2V2TTi65PclmT3LZxr0zaXZjZtPUlemtn07W3xsSQ/X1VrprofPNW2tqoeO7X5T0n+fAnH/GiSn6uq3aZjPryqvm+R+35PZsH71qp6aJLnzdm2uet1cWafxwOq6oFJXpBtvy73qKq9uvvy7n5Dkq9kdh/5YzIbmT4lyR9nNs19Ma5P8pgpJCfJd92bvwV7JPnytPzyBerd0N1vTvLpzGZPbMtnAgAAsE22GLS7+y+TnJXk6iTnZRZmklnwfUVVXZ3kuiSHzdntS0k+NbU/urv/OcmFSfapzTwMbfInSV4wtXlakmOTHFlV12QWgF+z1De4ifdMtV0z1f3TU21HZjYNfkNmI72LfoJ5d/9Zkg8k+eS0/znZ8hcKG/e9OrPpzZ+bjnHJnM2nJTm/poehzdnnLzO7V/xTSS5P8p7u3ty08a3x1po9FO7azL7ouDrJi5JcO00Jf0KS9y3mQN39zSTHZPY+rszsy4Nbk6Sq1lfVe7ZwiLckOamqrsrmR8iPmx7edk2Sbyc5b1s+EwAAgG1V3Ssxq5udRVXt1t23T08hf0eSL3T3b690XZt61Qkn9TtP/OWVLgMAANh+1ZabzCzqv/eCbfDKaST8usymgrtvGgAA2KGNeGDVklXVkfnuaeCXdPerF7HvCUkO32T12d194qj6tlVVPSTJBfNsemZ3/+NqP99STKPX290INgAAwL1lRYJ2d783yXu3ct8Tk2w3oXo+U7jd3BPZV/35AAAA2DxTxwEAAGAgQRsAAAAGErQBAABgIEEbAAAABhK0AQAAYCBBGwAAAAYStAEAAGAgQRsAAAAGErQBAABgIEEbAAAABhK0Icl+D99jpUsAAAB2EII2AAAADCRoAwAAwECCNgAAAAwkaAMAAMBAgjYAAAAMJGgDAADAQII2AAAADCRoAwAAwECCNgAAAAwkaAMAAMBAgjYAAAAMJGgDAADAQII2AAAADCRoAwAAwECCNgAAAAwkaAMAAMBAgjYAAAAMJGgDAADAQII2AAAADCRoAwAAwECCNgAAAAwkaAMAAMBAgjYAAAAMJGgDAADAQII2AAAADCRoAwAAwECCNktSVQ+qqmOW8XwHV9Vnpp+rq+oFm9RyTlV9rqr+qqqePK1/cFV9rKq+MP3+3uWqFwAAQNBmqR6UZNFBu2a2pZ9dm2R9d69L8twk766qNdO2tyU5v7sfn+SAJH81rT8+yQXdvXeSC6bXAAAAy0LQZqlOTrLXNML81qp6fVV9uqquqapfT5KqWltV11fV+zILyo+sqtun9tdV1f+ZRqo/UVU3VNWhmztZd9/R3d+ZXu6apKdz7JHk6Ul+d2r3re7++tTusCRnTMtnJPmJ+Y5dVUdV1RVVdcVXv/rVbbooAAAAGwnaLNXxSf56GmH+WJK9kxycZF2Sg6rq6VO7vZOc2t37dvfNSR6Y5OPdvW+S25K8Kcl/SPKCJG9c6IRV9aSqui7JhiRHT8H70Um+kuS9VXVVVb2nqh447fLQ7v67afnvkzx0vuN292ndvb671++5555bcSkAAAC+m6DNtnj29HNVkr9M8vjMAnaS3Nzdl81p+60k50/LG5L8eXd/e1peu9BJuvvyKaA/MckvV9WuSdYk+cEk7+zuA5N8I/NMEe/uzjQKDgAAsBwEbbZFJTmpu9dNP4/t7t+dtn1jk7bfnkJvktyd5M4k6e67MwvNW9Tdf5Xk9iRPSHJLklu6+/Jp8zmZBe8k+b9V9bAkmX7/w9LfGgAAwNYRtFmq25LsPi1/NMnPVdVuSVJVD6+q7xt5sqp69MaHn1XV92c2an5Td/99kr+pqh+Ymj4zyWen5XOT/Oy0/LNJ/nhkTQAAAAtZ1EgibNTd/1hVl1TVtUnOS/KBJJ+sqmQ22vwzSe4aeMqnJjm+qr6d2Uj4Md298cllxyY5s6rul+SGJEdO609O8sGqekWSm5O8aGA9AAAAC6p/mc0LO69TTz21jzlm2f57cAAAYPWpxTY0dRwAAAAGMnWc7UJVPSfJmzdZfWN3v2Al6gEAANhagjbbhe7+aGYPVwMAAFjVTB0HAACAgQRtAAAAGEjQBgAAgIEEbQAAABhI0AYAAICBBG0AAAAYSNAGAACAgQRtAAAAGEjQBgAAgIEEbQAAABhI0AYAAICBBG0AAAAYSNAGAACAgQRtAAAAGEjQBgAAgIEEbQAAABhI0AYAAICBBG0AAAAYSNAGAACAgQRtAAAAGEjQBgAAgIEEbQAAABhI0AYAAICBBG0AAAAYSNAGAACAgQRtAAAAGEjQBgAAgIEEbQAAABhI0AYAAICBBG0AAAAYSNAGAACAgQRtAAAAGEjQBgAAgIEEbQAAABhI0AYAAICBBG0AAAAYSNAGAACAgQRtAAAAGEjQBgAAgIEEbQAAABiounula4AV90u/9Eu33fe+971+petgx3H77bfvudtuu311petgx6FPMZo+xUj6E6Ntp33qq29605ueu5iGgjYkqaorunv9StfBjkOfYjR9itH0KUbSnxhttfcpU8cBAABgIEEbAAAABhK0Yea0lS6AHY4+xWj6FKPpU4ykPzHaqu5T7tEGAACAgYxoAwAAwECCNgAAAAwkaLNTqarnVtX1VfXFqjp+nu33r6qzpu2XV9Xa5a+S1WQRfeq1VfXZqrqmqi6oqu9fiTpZPbbUp+a0+6mq6qpatf/1Cfe+xfSnqnrR9OfUdVX1geWukdVlEX/vPaqqLqyqq6a/+56/EnWyOlTV71XVP1TVtZvZXlV1ytTfrqmqH1zuGreWoM1Oo6p2SfKOJM9Lsk+Sl1TVPps0e0WSr3X3Y5P8dpI3L2+VrCaL7FNXJVnf3fsnOSfJW5a3SlaTRfapVNXuSV6T5PLlrZDVZDH9qar2TvLLSX64u/dNctyyF8qqscg/o34lyQe7+8AkL05y6vJWySpzepLnLrD9eUn2nn6OSvLOZahpCEGbncnBSb7Y3Td097eS/O8kh23S5rAkZ0zL5yR5ZlXVMtbI6rLFPtXdF3b3HdPLy5I8YplrZHVZzJ9TSfIbmX0R+M/LWRyrzmL60yuTvKO7v5Yk3f0Py1wjq8ti+lQn+Z5peY8kf7uM9bHKdPdFSf5pgSaHJXlfz1yW5EFV9bDlqW7bCNrsTB6e5G/mvL5lWjdvm+7+TpJbkzxkWapjNVpMn5rrFUnOu1crYrXbYp+aps09srs/vJyFsSot5s+oxyV5XFVdUlWXVdVCI0uwmD71a0l+pqpuSfKRJMcuT2nsoJb6b63txpqVLgBgZ1BVP5NkfZIfWelaWL2q6j5JfivJy1e4FHYcazKbkvmMzGbcXFRV+3X311e0KlazlyQ5vbt/s6qenOT9VfWE7r57pQuD5WREm53Jl5M8cs7rR0zr5m1TVWsym/L0j8tSHavRYvpUqupZSU5Icmh337lMtbE6balP7Z7kCUk+UVU3JfmhJOd6IBqbsZg/o25Jcm53f7u7b0zy+cyCN8xnMX3qFUk+mCTd/ckkuybZc1mqY0e0qH9rbY8EbXYmn06yd1U9uqrul9kDOs7dpM25SX52Wn5hko93dy9jjawuW+xTVXVgkndnFrLd+8iWLNinuvvW7t6zu9d299rM7vs/tLuvWJly2c4t5u+9P8psNDtVtWdmU8lvWM4iWVUW06e+lOSZSVJV/z6zoP2VZa2SHcm5SV42PX38h5Lc2t1/t9JFLYap4+w0uvs7VfULST6aZJckv9fd11XVG5Nc0d3nJvndzKY4fTGzBzO8eOUqZnu3yD711iS7JTl7eq7el7r70BUrmu3aIvsULMoi+9NHkzy7qj6b5K4kr+9uM7mY1yL71H9N8r+q6hczezDayw1asDlV9QeZfdm353Rf/68muW+SdPe7MrvP//lJvpjkjiRHrkylS1f6PQAAAIxj6jgAAAAMJGgDAADAQII2AAAADCRoAwAAwECCNgAAAAwkaAMAAMBAgjYAAAAM9P8BF8uQVuxG6moAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1008x720 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# MXGB var importance\n",
    "mxgb_default.varimp_plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA9oAAAJTCAYAAAD3327jAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xm4JFV9//H3B0ZBFkERDeIyCkQEgZFNZROUGHXcQJCoiQLGBfddlLhGBX8kGlEBcUOjUYKKosgiArIpi8AwIIpGRsMiEhdkEdSZ7++POheapvveO0PN3IF5v56nn+4+derUqep773M/fU5VpaqQJEmSJEn9WGmmOyBJkiRJ0j2JQVuSJEmSpB4ZtCVJkiRJ6pFBW5IkSZKkHhm0JUmSJEnqkUFbkiRJkqQeGbQlSSuEJGcm+WsP7VyZ5OeLUX/DJJXk03d125Ik6e7BoC1JWiqSfKkFzFdOo+5Jre5uy6Jv9zTtS4RKssNM92VpW9wvOlYESXZtn//Jk9SZ+MLn50PlD0nyuiQnJFmQ5NYkv22/k8+ZYrv3SvLPSb6b5DdJ/tyeT0qyb5JZi7kfX259fMuY5eskuTrJLUk2H7H83kn2TvLN9nNyS5Kbk1yR5OutT6sNrTOrbXP4cWs7Hkcm2Xhx9mNZS/L+FeX3X7o7Waw/gJIkLYZPAS8A/hk4dFylJLOBXYFrgG8txf68ALjPUmxfujt6PfAm4BfAKcC1wGxgN+DvkhxcVW8dXinJw4BjgS2AXwPfbs9/Azwd+DvglUmeVVVXT7MvrwR2BP41yYlVdfHQ8iOA9YA3Dy9LsgnwNWBj4PdtX64AFgIPAXZq+/SB1sawAt438H5t4HHAi4HnJtl+RH8kaSyDtiRpqaiq05JcDjw2yZZVdcGYqi8BAnyuqu7y1O5J+vOrpdW2dDf2Q2CnqjpjsDDJY4Czgbck+VJVzRtYtgZwAvBo4LPAq6vqTwPLVwcOB/4R+E6SJwwuH6eqfp9kH+BE4ItJtqmqW1ubewO7A6cBHx7q6/rA9+hC/keAd1bVTUN1AjwFOHDM5hdV1XuGC5McBrwCeC3dl4aSNC1OHZckLU2fas8vHbUwycrAPnSjSZ8eKF8/ybuTnJ3k121K6lVtOvqdpnEOnged5FFJjk5yXZJFE9MpR52jnWSVJK9JcnySX7bpor9rU2H/frIdS7J2kkMHprJemuRV7R/6aUmyepJ3JJmX5KYkN7Z93mu6bUzR/pVJfp7kvkk+2t7/KcmFSZ7V6sxK8s4kP2v78fMk+41oa2J68r8k2T7J95L8sT2OT7LlmD6sneRDA+3/Lt005SdNsY3HJ/lOq19J/jFJAesDGwxN8x382dm9/Zz8bOCYnp/k1Unu9H9Pki+2Nh6a5JVJLmn9/HWSw5Pcd8x+PTTJxwb267dJzk1ywJi6hyb5RW6fmv3NJFtN9vktC1X11eGQ3covAb7a3u48tPjNdCH7dOCfh0N0C7l7A+fQjXi/djH6813gE8BmdKPPE7NeDgGuB15cVTW02oF0IfsLVfXG4ZDd2q2qOhHYdrp9aU5qz+sOL0iyavv9vaT9Xv0xyelJ9hjXWJJ/SHJGq/unJBcneVuSe4+oOyfJUbl9Sv91SX6U5CPp/naS5Epg4mfujIHfiaX2paWk6XFEW5K0NH2e7p/l5yd5U1XdPLT8aXTB6btVdcVA+S7AW4FTgQuBm4CNgOcBz0yyXQsCw/4WOBf4MfBFYDXghkn6ty7wH3Qjd98FrqObVvos4Pgk+1bVkSPWW4VuauoawH+193sCH299eN0k2wQgyf3a/m0B/IhuZHAl4KnAV5I8etQI2xJYBTgZuC/wjfb++cDXk+wKvAHYEjge+Evbj0OT/Kaqvjaive2Ad9EFkI/TfS67A09MsmtVnT2wj/cHzqKbznsu3dTedek+x5OTvKyqRl0kboe2jdOBzwAPBH4GvBd4I/BXuuA1YXC2xP8DbqUbqb0KWAt4MvAxYCu6L3ZG+Xe66c7fphtRfTLwcmCDVn6bJI+jO173oxth/TqwOrBJ6/cHBupu3dq7H90o8MQx2A14apJnVtVJA/Vn0X0OC6tqpv9P+0t7Hg5tE1+cvX9E6AWgqhYm+SDwTeBlwIcWY7tvpTud5A1JvkP3ua8J/OPwzJR0o+v/0N6+d6qGl2DWzK7t+fyh7a5C9zdjB7q/Nx+n+xnYEzg6yb9W1buG1vl/wFvo/s58ke7v2lzgIOApSZ5aVX9pdR8L/IBu6vuxwAK63+GNgFcBb2/LPgw8h27K/eeAieOzaDH3U1LfqsqHDx8+fPhYag/gKLoR671HLPtmW7bHUPmDgDVG1H8s3T+n3xoq37C1U8D7xvTjTOCvQ2WrAuuPqLs2cBndP8SrDC27sm3n+8C9B8ofQHdOaAHbjejbp4fa+WIrf+NQ+X3o/oFfBGw2zWN8ZmtrhzF9/cbgftB9kVHA7+gC6VoDyzaiC1jnDbW168AxfsXQsue28p8AGSj/TCs/dKj+xnRfgNwCPHTMNl4yZl+vBH4+ybHYYETZSsCXWrtbjfkcrgAeMlB+L7ovYArYcqB8FbowU8DzRmxruI1fAH8a8dk8hO66BFcO/RzNam3/ddw+jtjmxHH7BfCeMY9DWp2xx27E78B1dGFuo4HyR7R2/szQ78aINtZo6xfwN9Pdn7bu1u3n8Ja2/lFj6j1p4vNbnPZHHO9FQ8frw3S/V4vo/k6tMbTeO9t6xwKzBsr/Bvjftt7jBsp3bPUXAA8c2v532rK3DpR/tJXNHdHn+3PH37P3M+L334cPHzP7mPEO+PDhw4ePe/aDbmSwgDOHytdr/0hfC9xrMdr7DnAzsPJA2USYvWowtAytd6egPcV23spQaG7lE+H1CSPW+ee27FMj+vbpgbIHtgDygzHb3qqt88Fp9nWqoP3wEetMhMWdRiw7g25UePCf+Ykwd9lg+Yg+bN/er0IXMK8H1h5R/8BW/x0jtnHeJPs6adCeZL1th7fXyieC9t4j1nkpQ18sAHu1sq9NY5sTX0AcOGb5m9rypwyVbww8ajH2bfALiqkeUx47umsmfL3V/+jQsu1a+ZXT7Nv/MfRlxWLs18Rn80fgfmPqvIARf18Glu/Lnb902Hxg+UTQHveYD/zDiHavoPsd3nDEspe3dY8YKPtcK9t3RP1H0wXzywfKJoL2k6ZxnAzaPnwsh4+ZnpIkSbrnOwX4H2D7Nh36sla+D90/uUdWmy45qJ1D/HK60LkOdz7d6f50I26DLqqqPy9O55JsRjedcwfgwXQBcdD6I1abmJo87LT2/NgpNrst3ShrkrxnxPKJPjx6inam4/+q6pcjyq8GHsodp11PuAq4N90XAtcOLTujqmrEOqcB29Pt+1l006hXBc6pqj+MqH8KsD+jj9W5I8qmJckD6D7Pp9ONvq4+VGXU5wlDU4Ob/23P9xsoe3x7Pn4a3XlCe37EmM/5Ue350dx+LjBV9ZNptD3K96pq11ELkmxIN/1+Oj5KN7X9NLpjucyluxjbxLnOa9KdUvHlJWhqX7qfy0E/B4avIH6HqfptSvqmdFPev9z+dr27Lbsf3ZXZf1lVo241d0p7HvzZ3nJo2W2q6rIk1wAbJVmjqm4EvgK8GvhWkq/Snf5xdlX9z1Q7LGn5YNCWJC1VVTVxsaoD6UZ835QkdFcbL26/YNptkrwJ+De6qc0nA7+kGx0tuvOBN+POgRi62wtNW5LtW/sr0V21+Jt0U5oX0f1j/Mwx27luTNic2P5aU2x6nfb8uPYYZ40p2pmO68eU/5UuXNw4Zhl0U5+HDQfvCcP7PvF8zZj6E+VrT9LWYmnnhJ8PPJzuQlxfoPsZ+ivdFzOvYfTnCTDqy4CJ47DyQNlEf6+aRpcmPuepLm7Xx+fciyQfpjtOpwLPGPHF1cRns26SVapdFXxMW6tz+5cU073FF+3CYF+k+6xeTRd2P5Hk+3XnW4VN9OfBo9qqqtvuLZ3kIOBt0+lD+704J8nudLMo3p7kk237S/KzPZ11Htzq3VhVP0iyE/AOumsavKjtw0+A91TVUdPZD0kzx6AtSVoWPkd3j9oXJXk73fmKjwROGR4RSnIv4N10/5hvWVXXDi3fcZLtjAq/k3kn3ajrjlV15tB23kkXtEdZN0lGhO2/ac/jwi1Dy0feo3g596Ax5cP7fv1Q+bD1huoNWtzPccLL6EL2O6vq/YML2s/Na5aw3UETgXzcyPigiX2bW1Xf6WHbS0378usQumB7MvCsGnFLrqr6RRt9XY/u3tTfnaTZJ9F9ifWLqlqcL0/eT3eRwEOr6hNJFgGH0p3z/7ShuufSnYLyiCSPqDteVPEuq6rfJfkZsDndCPXVLNnP9uA6o2aY3GmdqjoLmNsuvLY13b6/mm6E/dqqOm3x9kbSsuTtvSRJS10Ly8fSXTDsOdx+P9ojRlR/EN1U0TNHhOz7MvW07MWxIfCb4ZDdPHGS9Vbh9inEg3ZuzxdOsd1z6MLkZF8aLK92bKFs2M7teWLfL6ObYv/YMbfI2qU9j7u/+jgLueMI86AN2/Ooq6VP9nkujolTBoYD32R1l+vPuX2eh9OFuBOAZ44K2QMmrhT/jjE/C6S7ldo72ttRv+fj+rIT3bnrl9OmrVfVYXRXbn9qklcM1m8jzxOju3e4ynePJkblV2rb/D1dWH5YkkeOqD/qZ3vi92Ln4cpJHkUXtH82aoZJVd1aVWdV1b/Q3SUgwLMHqixsz+N+LyTNAIO2JGlZmZgi/ia68z//DzhmRL1r6ALaNm3qKXDbdNKPccfzZe+qBXSj05sOFiZ5Od1F3CZz0OC9b9u5wRP3s/3cZCtW1TV052A+PsnbJ+6JO9SHDZM8fOpdWOY2pjt3/jZJnkt3HuxP6a7UTVXdQnfrs7XoZjMM1t+ILtT9mW6K8OL4LfDANso3bEF73nloe1szzSnD0/ANunO3d0+y5/DCJA8ZeHtM69NrM+a+7Em2S7LqUNnGLXwtdS0Qf4ZuNsC3gee0z24yB9N91jsDnxzR/9Xoblf3eGAed7wV22R9uS/dLQEX0d3Ka/B2gPsCvwf+LckGQ6u+nW4K+d5J/q1tf5SpTukY1ac96K5lMHxdhonb8R2cgfuzJ3kgt/8d+OxQfYB3JllnoP4sulvLhe5zmCi/089FMzGjZPDY/LY9P2yauyVpGXDquCRpWTmJLnRs295/fNSFy6q7/+7HgDcD85McSzeC/CS6f5S/T3+jkx+hC9RnJ/lvuqsbb0t3Eauv0V01epQr6UbdLxno3x5000IPqYF7SU9iP7oR2A/SBYQzuf0+3pvQTRXdk9HTTGfS8cAhSebSXZF54j7af6K7JdfgtO+Ji8y9Lsm2dJ/dxH201wD2q6H7Ik/D9+hmNZyQ5Ay6sH5hVR0HHEn3Rc7H2j3Cf053X/Nn0H2eU50rPaWqurUF7BOA/05yKt305fvQXdRsJ7rTESbq7t7qnpDkLOAiumP1MGAbugu2rUt3G6uJ4HUZ3Sjlsvg/7b10Fya8me4CYW8fMUh9QVUdO/Gmqm5I8lS6WSovBZ6R5Hi6sPs3dPeGfhDdiO5Uo+ODPkZ3kbH3VNV5gwuq6uokr6L78ubzSXaqqkVt2ZVJnkx3pfQ3Afsm+R7dlcEXtT5tT/f7di3dlwTDVhq6YN3qwGOAiS9I9q+qwYsvfojuAm27A/Pa/k/cR3tdujsG3BbMq+r0dv77G4FL2wXObm7HahO6340PD7T/DrrZI2e0/bip9edpdNcdGLy2xSl0M2Q+lGQLutMbFlXVB0fsp6RlZaYve+7Dhw8fPlacB91Iz8Rtc8bevoguYLyFLnD8iW6U+wt0I0sTt/wZvF/xyHtVD7U58vZewLPopnLfQDdidiJdOJy4Vdc/DtW/ki7ArQ0cRnfO5q3Aj+lGaTNUf2zf6AL6a4Ef0J2beQtdsD4ZeB1w/2ke18lu7zXydk7jjkdbNuoYT9xC6l/oQssp7ZjdQBcktxrT1v3oRkB/3o7TxDHedUTd27Yxyb6uAXyS7mJkfx0+tnRh5Nt0X1rcRHdxtH3HfQ6j9nU6/aE7F/xwui+PbqWbofFDukA2XPdBdMHsUrpwdSPdFcCPBl7IHW9Vd1fuo33yJHUm9v/nQ+UT+z/ZY+TvFd2V6V9G9+XHdXTnSl9Hd972Sxi4v/Q09mH3tq1zJluPbiZIjTnO96b70uBb7efjVrq/HwvoZiLsC6w+4m/NqH3+C93fnW8ATx7Tl/vQ/T5cSve7ewPdrfHudH/1gXVeSHdV/on7yF9CNyK/ylC9p9J9cXQZ3d+GG+nuU/9R4GEj2n0x3eyBiYtGTvvnx4cPH0vnkaolvd6IJElaUbQR4u8y4kJjkiTpjjxHW5IkSZKkHhm0JUmSJEnqkUFbkiRJkqQeeY62JEmSJEk98vZeEvD5z3++XvziF890NyRJkiQtv+50/8NxnDouATfddNNMd0GSJEnSPYRBW5IkSZKkHhm0JUmSJEnqkUFbkiRJkqQeGbQlSZIkSeqRQVuSJEmSpB4ZtCVJkiRJ6pFBW5IkSZKkHhm0JUmSJEnqkUFbkiRJkqQeGbQlSZIkSeqRQVuSJEmSpB4ZtCVJkiRJ6pFBW5IkSZKkHhm0JUmSJEnqkUFbkiRJkqQeGbQlSZIkSeqRQVuSJEmSpB4ZtCVJkiRJ6pFBW5IkSZKkHhm0JUmSJEnqkUFbkiRJkqQeGbQlSZIkSeqRQVuSJEmSpB4ZtCVJkiRJ6pFBW5IkSZKkHhm0JUmSJEnqkUFbkiRJkqQeGbQlSZIkSerRrJnugLQ8mH/V9cze/7iZ7oYkSZIkYMFBc2e6C3eJI9qSJEmSJPXIoC1JkiRJUo8M2pIkSZIk9cigLUmSJElSjwzakiRJkiT1yKAtSZIkSVKPDNqSJEmSJPXIoC1JkiRJUo8M2pIkSZIk9cigLUmSJElSjwzakiRJkiT1yKAtSZIkSVKPDNqSJEmSJPXIoC1JkiRJUo8M2pIkSZIk9cigLUmSJElSjwzakiRJkiT1yKA9iSQ3LqPt7J3kwctoW6cl2XpZbGtou7OTvGDg/dZJDlkabY9YPiP7LEmSJGnFZNBePuwNLJOgvTQlmTXJ4tnAbWG4qs6vqtf2tOk7tC1JkiRJM8mgPQ3pHJzkkiTzk+zVytdI8r0kF7TyZ7fy2UkuS/KpJJcmOSnJfca0vQewNfClJBcluU+SJye5sLX52SSrjFn3qUmOHni/c5Jvt9eHJTm/bf+9Y9a/ceD1HkmObK/XTfK1JOe1x/aTHJv3JPnPJGcB/9n2/Yx2TC5Isl2rehCwY9vHNwz19f5JvpHk4iQ/TLL5JNt7YmvjonaM1hzR9n2SfKV9BscA4479y9oxOn/hzdeP26QkSZIkLRaD9vTsDswBtgB2BQ5Osh5wC7BbVW0J7AL8e5K0dTYCPlFVmwJ/AJ47quGq+ipwPvDCqpoDFHAksFdVbQbMAvYb06+TgcclWb293wv4Snt9QFVtDWwOPHGy8DrCR4GPVNU2rd+fnqL+JsCuVfV84DfA37VjshcwMT18f+CMqppTVR8ZWv+9wIVVtTnwDuALk2zrzcCr2rHaEfjTiLb3A26uqkcD7wa2GtVQVR1RVVtX1dYrr7bWFLsoSZIkSdNj0J6eHYAvV9XCqroW+D6wDRDgg0kupgu96wMPautcUVUXtdc/opvePB2Paute3t5/HthpVMWq+itwAvDMNm17LvDNtvh5SS4ALgQ2pQvD07Ur8PEkFwHHAvdNssYk9Y+tqj+11/cCPpVkPnD0NLe7A/CfbZ9OAdZJct8xdc8CPpzktcDa7RgM2wn4YmvvYuDiafRBkiRJknox2Tm1mtoLgXWBrarqL0kWAKu2ZbcO1FvImOnLPfgK8Grgd8D5VXVDkkfQjfxuU1W/b1PCVx2xbg28Hly+EvD4qrplmn24aeD1G4Br6Ub/V6Ib9e9NVR2U5Djg6cBZSf6+z/YlSZIk6a5yRHt6zgD2SrJyknXpRkzPBdYCftNC9i7Aw5ew/RuANdvrnwKzk2zY3v8T3Qj6ON8HtgReyu3Txu9LF36vT/Ig4Glj1r02yaOTrATsNlB+EvCaiTdJ5izGvqwFXFNVi1rfV27lg/s47Ay6Ly1IsjPwf1X1x1EVk2xQVfOr6kPAecDGI9o+nXZxtCSPoZs+L0mSJEnLhEF7eo6hm348DzgFeGtV/Rr4ErB1myb9IuAnS9j+kcDhbap2gH2Ao1u7i4DDx61YVQuBb9OF6W+3snl0U8Z/AvwX3XTrUfZv65wNXDNQ/tq2Xxcn+THwisXYl0OBFyeZRxeCJ0a7LwYWJpmX5A1D67wH2KpNwT8IePEk7b++XZTuYuAvwPEj2j4MWCPJZcD76KbuS5IkSdIykaqaupZ0D7ffAQfW8Qsd+JYkSZKWBwsOmjvTXRglU1fpOKItSZIkSVKPvBjaMpTkE8DwPak/WlWfm8a6xwCPGCp+W1Wd2Ff/ptj+PsDrhorPqqpX3RO2J0mSJEl9MWgvQ3clJFbVblPXWnralwFTfiFwd92eJEmSJPXFqeOSJEmSJPXIoC1JkiRJUo8M2pIkSZIk9cigLUmSJElSjwzakiRJkiT1yKAtSZIkSVKPDNqSJEmSJPXIoC1JkiRJUo8M2pIkSZIk9WjWTHdAWh5stv5aHPbKuTPdDUmSJEn3AI5oS5IkSZLUI4O2JEmSJEk9MmhLkiRJktQjg7YkSZIkST0yaEuSJEmS1CODtiRJkiRJPTJoS5IkSZLUI4O2JEmSJEk9MmhLkiRJktSjWTPdAWl5MP+q65m9/3Ez3Q1pubbgoLkz3QVJkqS7BUe0JUmSJEnqkUFbkiRJkqQeGbQlSZIkSeqRQVuSJEmSpB4ZtCVJkiRJ6pFBW5IkSZKkHhm0JUmSJEnqkUFbkiRJkqQeGbQlSZIkSeqRQVuSJEmSpB4ZtCVJkiRJ6pFBW5IkSZKkHhm0JUmSJEnqkUFbkiRJkqQeGbQlSZIkSeqRQVuSJEmSpB4ZtCVJkiRJ6pFBewWUZHaSFyyjbe2d5OPLYDuvT7LaTPZBkiRJksCgvdxKsvJSancWMBtYJkF7GXo9MDJoS5IkSdKyZNCeAW1E+SdJvpTksiRfTbJakgVJPpTkAmDPJHOS/DDJxUmOSXK/tv5pST6a5KIklyTZtpWvnuSzSc5NcmGSZ7fyvZMcm+QU4HvAQcCObf03JDk9yZyB/p2ZZIsR/V6p9XHtgbKfJXlQkmcmOadt9+QkDxqx/pFJ9hh4f+PA67ckOa/t63snOXarJzkuyby273sleS3wYODUJKe2evskuTzJucD2Y9p6WZLzk5y/8Obrx21SkiRJkhaLQXvmPAo4tKoeDfwReGUr/21VbVlVXwG+ALytqjYH5gPvHlh/taqa09b7bCs7ADilqrYFdgEOTrJ6W7YlsEdVPRHYHzijquZU1UeAzwB7AyT5W2DVqpo33OGqWgR8E9it1X0c8MuquhY4E3h8VT0W+Arw1ukeiCRPATYCtgXmAFsl2WlM9acCV1fVFlX1GOCEqjoEuBrYpap2SbIe8F66gL0DsMmohqrqiKrauqq2Xnm1tabbXUmSJEmalEF75vxvVZ3VXn+RLhACHAWQZC1g7ar6fiv/PDAYPr8MUFWnA/dto8xPAfZPchFwGrAq8LBW/7tV9bsxfTkaeEaSewH7AkdO0u+jgL3a63+Y6C/wEODEJPOBtwCbTtLGsKe0x4XABcDGdMF7lPnA37WR/x2ratRQ9OOA06rquqr680AfJUmSJGmpM2jPnBrz/qa7sH6A57aR6jlV9bCqumyqdqvqZuC7wLOB5wFfmmS7PwA2TLIu8Bzg6638Y8DHq2oz4OV0IX/YX2k/c0lWAu7dygMcONDvDavqM2P6ejnd6Px84P1J3jVJXyVJkiRpmTNoz5yHJXlCe/0CuqnXt2kjtb9PsmMr+ifg+wNV9gJIsgNwfat/IvCaJGnLHjtm2zcAaw6VfRo4BDivqn4/rtNVVcAxwIeBy6rqt23RWsBV7fWLx6y+ANiqvX4WcK/2+kRg3yRrtH6vn+SBoxpI8mDg5qr6InAwXege3qdzgCcmWaeN0u85bn8kSZIkqW+zZroDK7CfAq9K8lngx8BhwGuG6rwYOLzdtuoXwD4Dy25JciFdWN23lf0r8B/AxW3E+ArgGSO2fTGwMMk84Miq+khV/SjJH4HPTaPvRwHn0c7rbt4DHJ3k98ApwCNGrPcp4JttuyfQRtmr6qQkjwZ+0L4juBH4R+A3I9rYjO7c80XAX4D9WvkRwAlJrm7nab+HbvT9D8BF09gnSZIkSepFugFKLUtJZgPfbhfzWpL1TwPeXFXn99inB9Od171xu+jZCmW/Aw6s4xduPtPdkJZrCw6aO9NdkCRJmkmZbkWnjoskL6Kbbn3AihiyJUmSJKlPTh2fAVW1AFii0ey2/s69daZr7wt0txK7TZJ9gNcNVT2rql7V57bHSbIO3T2/hz154LxwSZIkSVruGLQ1UlV9jumdr720tv9buntqS5IkSdLdilPHJUmSJEnqkUFbkiRJkqQeGbQlSZIkSeqRQVuSJEmSpB4ZtCVJkiRJ6pFBW5IkSZKkHhm0JUmSJEnqkUFbkiRJkqQeGbQlSZIkSeqRQVuSJEmSpB7NmukOSMuDzdZfi8NeOXemuyFJkiTpHsARbUmSJEmSemTQliRJkiSpRwZtSZIkSZJ6ZNCWJEmSJKlHBm1JkiRJknpk0JYkSZIkqUcGbUmSJEmSemTQliRJkiSpRwZtSZIkSZJ6NGumOyAtD+ZfdT2z9z9upruh5dyCg+bOdBckSZJ0N+CItiRJkiRJPTJoS5IkSZLUI4O2JEmSJEk9MmhLkiRJktQjg7YkSZIkST0yaEuSJEmS1CODtiRJkiRJPTJoS5IkSZLUI4O2JEmSJEk9MmhLkiRJktQjg7YkSZIkST0yaEuSJEmS1CODtiRJkiRJPTJoS5IkSZLUI4O2JEmSJEk9MmhLkiRJktSjFT5oJ3lPkjdPsvzIJHuMKJ+T5OlTtL1zku2WsF+vSPKiJVl3eZTk9UlWG3j/nSRrL422h5btneTjfWxHkiRJkqZjhQ/ad8EcYNKgDewMLFHQrqoVb0kLAAAgAElEQVTDq+oLS7LuTEhnsp+n1wO3heGqenpV/aGnzd+hbUmSJEmaSStk0E5yQJLLk5wJPKqVbZDkhCQ/SnJGko0HVtk1yfltnWckuTfwPmCvJBcl2WvENmYDrwDe0OrsmGR2klOSXJzke0keNkkfbxtpT3Jakg8lObf1YcdWvnKSf0tySWvzNa38yUkuTDI/yWeTrNLKFyQ5sPXn/CRbJjkxyf8kecXAtt+S5LzW5nsn6ePsJD9N8gXgEuChSQ5rbV86sW6S1wIPBk5NcupAXx7QXr+x7cMlSV4/yfZWT3Jcknmt7l5j2t6nHadzge0nae9lra/nL7z5+nHVJEmSJGmxrHBBO8lWwD9w+4j0Nm3REcBrqmor4M3AoQOrzQa2BeYCh9Mdt3cBR1XVnKo6ang7VbWg1f1Iq3MG8DHg81W1OfAl4JDF6PqsqtqWbvT23a3sZa1vcybaTLIqcCSwV1VtBswC9hto51dVNQc4o9XbA3g8MBGKnwJs1PZ3DrBVkp0m6ddGwKFVtWlV/RI4oKq2BjYHnphk86o6BLga2KWqdhlcuX0e+wCPa/14aZLHjtnWU4Grq2qLqnoMcMJw20nWa/uyPbADsMm4jlfVEVW1dVVtvfJqa02yi5IkSZI0fStc0AZ2BI6pqpur6o/AscCqdFO8j05yEfBJYL2Bdf67qhZV1c+AXwAbDzc6TU8A/qu9/k+6IDhdX2/PP6IL1wC7Ap+sqr8CVNXv6Ebor6iqy1udzwODQfnY9jwfOKeqbqiq64Bb2znTT2mPC4EL6PZ1o0n69cuq+uHA++cluaCtvymTBN1mB7rP46aqurHt545j6s4H/q6N7u9YVaOGoR8HnFZV11XVn4E7fQkiSZIkSUvTrJnuwHJiJeAPbaR3lJri/bJwa3teyF373CbaWTTweuL9LCDAgVX1yWm2d9PEiySPoJsNsE1V/T7JkXRfYvSiqi5PsiXdTIT3J/leVb2vr/YlSZIkqQ8r4oj26cBzktwnyZrAM4GbgSuS7Am3Xdhri4F19kyyUpINgEcCPwVuANacYlvDdc6mm7YO8EK66dt3xXeBlyeZ1fp9/9a32Uk2bHX+Cfj+YrR5IrBvkjVam+sneeA0170vXfC+PsmDgKcNLBt3vM6g+zxWS7I6sBtjjkuSBwM3V9UXgYOBLUe0fQ7dlPV1ktwL2HOafZckSZKkXqxwQbuqLqCbTjwPOB44ry16IfCSJPOAS4FnD6z2K+DcVv8VVXULcCqwybiLoTXfAnabuBga8BpgnyQX0wXg193F3fl069vFrd8vaH3bh24a/Hy6kerDp9tgVZ1EN739B239rzL1FwoT686jmzL+k9bGWQOLjwBOmLhg2cA6F9CdK34uXUj+dFVdOGYTmwHntun97wbeP9x2VV0DvAf4Qdv+ZdPpuyRJkiT1JVUzMQtaWr7sd8CBdfzCzWe6G1rOLTho7kx3QZIkSTMn0624wo1oS5IkSZK0NHkxtB4k2Yc7TwM/q6peNY11D+DO5xEfXVUf6Kt/d1WSdYDvjVj05Kr67d19e5IkSZLUJ4N2D6rqc8DnlnDdDwDLTagepYXbcVdkv9tvT5IkSZL65NRxSZIkSZJ6ZNCWJEmSJKlHBm1JkiRJknpk0JYkSZIkqUcGbUmSJEmSemTQliRJkiSpRwZtSZIkSZJ6ZNCWJEmSJKlHBm1JkiRJknpk0JYkSZIkqUezZroD0vJgs/XX4rBXzp3pbkiSJEm6B3BEW5IkSZKkHhm0JUmSJEnqkUFbkiRJkqQeGbQlSZIkSeqRQVuSJEmSpB4ZtCVJkiRJ6pFBW5IkSZKkHhm0JUmSJEnqkUFbkiRJkqQezZrpDkjLg/lXXc/s/Y+b6W6oJwsOmjvTXZAkSdIKzBFtSZIkSZJ6ZNCWJEmSJKlHBm1JkiRJknpk0JYkSZIkqUcGbUmSJEmSemTQliRJkiSpRwZtSZIkSZJ6ZNCWJEmSJKlHBm1JkiRJknpk0JYkSZIkqUcGbUmSJEmSemTQliRJkiSpRwZtSZIkSZJ6ZNCWJEmSJKlHBm1JkiRJknpk0JYkSZIkqUcGbUmSJEmSemTQXo4luXGm+7AsJZmd5AVLsN7aSV45yfIjk+xx13onSZIkSdNj0NbyZDaw2EEbWBsYG7QlSZIkaVkyaN8NpHNwkkuSzE+yVytfI8n3klzQyp/dymcnuSzJp5JcmuSkJPeZpP2XJjkvybwkX0uyWis/MslhSX6Y5BdJdk7y2db2kQPrP79t/5IkHxoov3Hg9R4T67R2D0lydmt3YrT5IGDHJBclecOYvm6a5NxW5+IkG7X1NmhlB7fj9fEkP01yMvDAMW29LMn5Sc5fePP10/gkJEmSJGlqBu27h92BOcAWwK7AwUnWA24BdquqLYFdgH9PkrbORsAnqmpT4A/Acydp/+tVtU1VbQFcBrxkYNn9gCcAbwCOBT4CbApslmROkgcDHwKe1Pq4TZLnTGOf1gN2AJ5BF5QB9gfOqKo5VfWRMeu9AvhoVc0BtgaubOv9T1vvLcBuwKOATYAXAduNaqiqjqiqratq65VXW2saXZYkSZKkqRm07x52AL5cVQur6lrg+8A2QIAPJrkYOBlYH3hQW+eKqrqovf4R3bTscR6T5Iwk84EX0gXpCd+qqgLmA9dW1fyqWgRc2trcBjitqq6rqr8CXwJ2msY+faOqFlXVjwf6PB0/AN6R5G3Aw6vqTyPq7MTtx+tq4JTFaF+SJEmS7hKD9t3bC4F1ga3aCO+1wKpt2a0D9RYCsyZp50jg1VW1GfDegTYG21k01OaiKdoEqIHXqw4tG2wrTFNV/RfwLOBPwHeSPGm660qSJEnSsmDQvns4A9grycpJ1qUbsT0XWAv4TVX9JckuwMOXsP01gWuS3IsuvC+Oc4EnJnlAkpWB59ONuANcm+TRSVaim849lRtaX8ZK8kjgF1V1CPBNYPMR653O7cdrPbpp9ZIkSZK0TEw1IqnlwzF050nPoxslfmtV/TrJl4BvtSnf5wM/WcL23wmcA1zXnicNu4Oq6pok+wOn0o1MH1dV32yL9we+3do9H1hjiuYuBhYmmQccOeY87ecB/5TkL8CvgQ9W1e+SnJXkEuB44K1054z/GPgV3XRzSZIkSVom0p1+K63Y9jvgwDp+4eYz3Q31ZMFBc2e6C5IkSbrnmfYpr04dlyRJkiSpR04dX4Ek+QSw/VDxR6vqczPRn8kk+Xu624YNuqKqpnOutyRJkiTNGIP2CqSqXjXTfZiuqjoROHGm+yFJkiRJi8up45IkSZIk9cigLUmSJElSjwzakiRJkiT1yKAtSZIkSVKPDNqSJEmSJPXIoC1JkiRJUo8M2pIkSZIk9cigLUmSJElSjwzakiRJkiT1yKAtSZIkSVKPZs10B6TlwWbrr8Vhr5w7092QJEmSdA/giLYkSZIkST0yaEuSJEmS1CODtiRJkiRJPTJoS5IkSZLUI4O2JEmSJEk9MmhLkiRJktQjg7YkSZIkST0yaEuSJEmS1CODtiRJkiRJPZo10x2Qlgfzr7qe2fsfN9PdWGEtOGjuTHdBkiRJ6o0j2pIkSZIk9cigLUmSJElSjwzakiRJkiT1yKAtSZIkSVKPDNqSJEmSJPXIoC1JkiRJUo8M2pIkSZIk9cigLUmSJElSjwzakiRJkiT1yKAtSZIkSVKPDNqSJEmSJPXIoC1JkiRJUo8M2pIkSZIk9cigLUmSJElSjwzakiRJkiT1yKAtSZIkSVKPDNq6TZKzZ7oPSyLJnCRPn2T5giQPWJZ9kiRJkrTiMmjrNlW13Uz3YQnNAcYGbUmSJElalgzauk2SG9vzzklOS/LVJD9J8qUkacu2SXJ2knlJzk2yZpJVk3wuyfwkFybZpdXdO8k3kny3jSq/OskbW50fJrl/q7dBkhOS/CjJGUk2nqSPeya5pG3/9CT3Bt4H7JXkoiR7JVknyUlJLk3yaSBj2npZkvOTnL/w5ut7PpqSJEmSVlQGbY3zWOD1wCbAI4HtW6g9CnhdVW0B7Ar8CXgVUFW1GfB84PNJVm3tPAbYHdgG+ABwc1U9FvgB8KJW5wjgNVW1FfBm4NBJ+vUu4O/b9p9VVX9uZUdV1ZyqOgp4N3BmVW0KHAM8bFRDVXVEVW1dVVuvvNpai3t8JEmSJGmkWTPdAS23zq2qKwGSXATMBq4Hrqmq8wCq6o9t+Q7Ax1rZT5L8Evjb1s6pVXUDcEOS64FvtfL5wOZJ1gC2A45ug+YAq0zSr7OAI5P8N/D1MXV2ogv3VNVxSX6/ODsuSZIkSXeFQVvj3DrweiFL/rMy2M6igfeLWpsrAX+oqjnTaayqXpHkccBc4EdJtlrCfkmSJEnSUuHUcS2OnwLrJdkGoJ2fPQs4A3hhK/tbuqnaP51Og21U/Ioke7b1k2SLcfWTbFBV51TVu4DrgIcCNwBrDlQ7HXhBq/804H6LtZeSJEmSdBcYtDVt7XzovYCPJZkHfBdYle6c6pWSzKc7h3vvqrp1fEt38kLgJa3NS4FnT1L34HbRtUuAs4F5wKnAJhMXQwPeC+yU5FK6KeS/WqwdlSRJkqS7IFU1032QZtx+BxxYxy/cfKa7scJacNDcme6CJEmSNJWRdzMaxRFtSZIkSZJ65MXQtFxKcgCw51Dx0VX1gZnojyRJkiRNl0Fby6UWqA3VkiRJku52nDouSZIkSVKPDNqSJEmSJPXIoC1JkiRJUo8M2pIkSZIk9cigLUmSJElSjwzakiRJkiT1yKAtSZIkSVKPDNqSJEmSJPXIoC1JkiRJUo8M2pIkSZIk9WjWTHdAWh5stv5aHPbKuTPdDUmSJEn3AI5oS5IkSZLUI4O2JEmSJEk9MmhLkiRJktQjg7YkSZIkST0yaEuSJEmS1CODtiRJkiRJPTJoS5IkSZLUI4O2JEmSJEk9MmhLkiRJktSjWTPdAWl5MP+q65m9/3Ez3Y0VyoKD5s50FyRJkqSlwhFtSZIkSZJ6ZNCWJEmSJKlHBm1JkiRJknpk0JYkSZIkqUcGbUmSJEmSemTQliRJkiSpRwZtSZIkSZJ6ZNCWJEmSJKlHBm1JkiRJknpk0JYkSZIkqUcGbUmSJEmSemTQliRJkiSpRwZtSZIkSZJ6ZNCWJEmSJKlHBm1JkiRJknpk0JYkSZIkqUcGbUmSJEmSemTQ7kmSs6dR5/VJVlsW/RnY5nOSbLIstzm0/SmPy1D9I5PssZjrjN3HJLOTXLI47UmSJEnSXWHQ7klVbTeNaq8HRgbtJCsv6banWPc5wDIP2klmwbSPy101I/soSZIkSaMYtHuS5Mb2vHOS05J8NclPknwpndcCDwZOTXLqxDpJ/j3JPOAJSZ6c5MIk85N8Nskqk2xvQZIPJbkA2DPJS5Ocl2Rekq8lWS3JdsCzgIOTXJRkg/Y4IcmPkpyRZONJtnFkksOTnJ/k8iTPaOUrJzm4be/iJC8f2PczkhwL/HjouKStc0nbv70Gyj+e5KdJTgYeOMVxPijJj9t2/23MPm7VjsM84FWTtPWytm/nL7z5+sk2K0mSJEnTNmumO3AP9VhgU+Bq4Cxg+6o6JMkbgV2q6v9avdWBc6rqTUlWBX4GPLmqLk/yBWA/4D8m2c5vq2pLgCTrVNWn2uv3Ay+pqo+10PvtqvpqW/Y94BVV9bMkjwMOBZ40yTZmA9sCG9B9SbAh8CLg+qrapn0ZcFaSk1r9LYHHVNUVQ+3sDswBtgAeAJyX5HTgCcCj6EakH0QX0D87qiNJ1gF2AzauqkqydlX9YcQ+Xgy8uqpOT3LwuB2rqiOAIwD2O+DAYuEkR0GSJEmSpskR7aXj3Kq6sqoWARfRhdVRFgJfa68fBVxRVZe3958HdppiO0cNvH5MG02eD7yQLujfQZI1gO2Ao5NcBHwSWG+Kbfx3VS2qqp8BvwA2Bp4CvKi1cQ6wDrBRq3/uiJANsAPw5apaWFXXAt8Htmn7OFF+NXDKJH25HrgF+EyS3YGbR+zj2sDaVXV6K/rPKfZPkiRJknrliPbScevA64WMP863VNVdGUe9aeD1kcBzqmpekr2BnUfUXwn4Q1XNWYxt1Ij3AV5TVScOLkiy81CfelVVf02yLfBkYA/g1Uw+Gi9JkiRJy5wj2svWDcCaY5b9FJjdpmYD/BPdqO90rQlck+RedCPad9pmVf0RuCLJnnDb+dFbTNHunklWSrIB8MjWzxOB/dq2SPK3SVafop0zgL3a+d3r0o1knwucPlC+HrDLuAbaiPxaVfUd4A1009CH9/EPwB+S7NCWvfBODUmSJEnSUmTQXraOAE6YuBjaoKq6BdiHblr3fGARcPhitP1OumncZwE/GSj/CvCWdpG1DeiC50vahcIuBZ49Rbu/ogvEx9Od230L8Gm6c6kvSHfrrE8y9eyIY4CLgXl008PfWlW/buU/a+19AfjBJG2sCXy7nYN9JvDGMfu4D/CJNrU9U/RLkiRJknqVquGZwVInyZEMXGTsnmy/Aw6s4xduPtPdWKEsOGjuTHdBkiRJWhzTHsRzRFuSJEmSpB55MbTlXJJjgEcMFb9t+EJkd3EbBwB7DhUfXVV797WNxbUs9luSJEmSlgaD9nKuqnZbBtv4APCBpb2dxbEs9luSJEmSlganjkuSJEmS1CODtiRJkiRJPTJoS5IkSZLUI4O2JEmSJEk9MmhLkiRJktQjg7YkSZIkST0yaEuSJEmS1CODtiRJkiRJPTJoS5IkSZLUo1kz3QFpebDZ+mtx2CvnznQ3JEmSJN0DOKItSZIkSVKPDNqSJEmSJPXIoC1JkiRJUo8M2pIkSZIk9cigLUmSJElSjwzakiRJkiT1yKAtSZIkSVKPDNqSJEmSJPXIoC1JkiRJUo9mzXQHpOXB/KuuZ/b+x810N26z4KC5M90FSZIkSUvIEW1JkiRJknpk0JYkSZIkqUcGbUmSJEmSemTQliRJkiSpRwZtSZIkSZJ6ZNCWJEmSJKlHBm1JkiRJknpk0JYkSZIkqUcGbUmSJEmSemTQliRJkiSpRwZtSZIkSZJ6ZNCWJEmSJKlHBm1JkiRJknpk0JYkSZIkqUcGbUmSJEmSemTQliRJkiSpRwZtSZIkSZJ6ZNDuSZL3Jdl1ijo7J9luWfWpbXN2khcsy20ObX/K4zJUf+ck317MbUy6j0lOS7L14rQpSZIkSUvKoN2TqnpXVZ08RbWdgZFBO8msJd32FOvOBmYkaCdZeZrH5a6azQztoyRJkiQNM2hz24joZUk+leTSJCcluU+SOUl+mOTiJMckud8kbRyZZI/2ekGS9ya5IMn8JBsnmc3/b+/Ooy0r6zOPfx8pFBksWunYNg6FgBoQKKDEEUUxqGBAVASjUZEWAZuINi4xuJyiYepoQ0QQB0AERWg1JAi0zSAEASkoqAJlUAaDMXGuZhKh+PUf5y24Ke9wbtVb99at+n7Wuuvus8+79/vb++x1i+e8797AAcD7klyXZMe2zYlJrgKOTvLEJN9u/V2ZZOtx+vtYktOSXA6c1o7hstbntSNGzo8Edmx9vi/JWkmOSXJ16+fd4/SxU5JLk5yb5OZW62Pae7skuaL1dVaS9Ucc+1FJrgX2Wua87JxkQTsnX07yuLb+1Uluatu8foLP6mXtWK5r+9pglGN8fJKvt8/0W8Djx9jX/knmJ5m/5L7F43UrSZIkSUMzaD9qc+D4qtoS+B3wBuArwAeramtgEfDRSezvV1W1HXACcGhV3QGcCHymquZW1WWt3VOBF1XV+4GPAwtaf3/d+h/PFsArq+rNwC+AP2t97g0c19ocBlzW+vwMsB+wuKqeBzwPeFeSTcbpYwfg4NbXpsDrk2wEfLj1vR0wH3j/iG1+XVXbVdXXl65Isg5wCrB3VW0FzAIObOu/APw5sD3wXyY45kOB91TVXGBH4P5RjvFA4L6q+lMGn9n2o+2oqk6qqnlVNW+tdWdP0K0kSZIkDceg/ajbq+q6tnwNg1C5YVV9r607FXjpJPb3zRH7mjNOu7OqaklbfglwGkBVXQQ8KckTxtn2nKq6vy2vDXwhySLgLAbBeDS7AG9Lch1wFfAkBl8yjOUHVXVbq/FrrcYXtP1f3vbzduAZI7Y5c5T9PJvBOb6lvV56Pp/T1t9aVQV8dZxaAC4HPp3krxh8Pg+N0ualS/dTVQuBhRPsU5IkSZK6We77gldDD4xYXgJs2Gl/Sxj/PN+7An2M3PZ9wL8D2zD4AuX3Y2wT4OCqumDIPmqU1wG+20bSJ6qrq6o6Msm5wK4Mgv6rVlZfkiRJkrQ8HNEe22Lgt0l2bK//EvjeOO2HcTewwTjvXwa8BQb3RzOYfv7/htz3bODnVfUwg1rXGqPPCxhM2V679fOsJOuNs98dkmzS7s3eG/hn4ErgxUk2a/tYL8mzJqjvZmDO0m149Hze1NZv2taPFd5pfW1aVYuq6ijgagYj4sse46W0h6MleS4w5r3ukiRJktSbI9rjeztwYpJ1gduAfVdwf/8InJ1kDwb3PS/rY8CXkywE7mv9D+tzwP9O8jbgfB4dVV4ILElyPYN7pI9lMJX92iQBfgm8bpz9Xg18FtgMuBj4VlU9nOQdwNeWPtCMwT3bt4y+C6iq3yfZFzirPSX9auDEqnogyf7AuUnuY/Blw3hfRhyS5OXAw8CNwHlteeQxngCcnORHwI8YTN+XJEmSpCmRwW2x0h9ro+qHVtVrp7uWle3Aw4+o85asOgPfdxy523SXIEmSJOk/yrANnTouSZIkSVJHTh2fpCTHAy9eZvWxVXXySupvX+C9y6y+vKre07GPrWhPOx/hgap6PnBJr34mYyqOW5IkSZJWBoP2JE110GsBfqWE+BF9LALmrsw+JmsqjluSJEmSVganjkuSJEmS1JFBW5IkSZKkjgzakiRJkiR1ZNCWJEmSJKkjg7YkSZIkSR0ZtCVJkiRJ6sigLUmSJElSRwZtSZIkSZI6MmhLkiRJktSRQVuSJEmSpI5mTXcB0qpgq41nc8JBu013GZIkSZJWA45oS5IkSZLUkUFbkiRJkqSODNqSJEmSJHVk0JYkSZIkqSODtiRJkiRJHRm0JUmSJEnqyKAtSZIkSVJHBm1JkiRJkjoyaEuSJEmS1NGs6S5AWhUs+tli5hx27rht7jhytymqRpIkSdJM5oi2JEmSJEkdGbQlSZIkSerIoC1JkiRJUkcGbUmSJEmSOjJoS5IkSZLUkUFbkiRJkqSODNqSJEmSJHVk0JYkSZIkqSODtiRJkiRJHRm0JUmSJEnqyKAtSZIkSVJHBm1JkiRJkjoyaEuSJEmS1JFBW5IkSZKkjgzakiRJkiR1ZNCWJEmSJKkjg7YmJcmGSQ6awv52SHJd+7k+yZ7L1HJ2kpuS/CjJC9v6Jyb5bpJb2+//NFX1SpIkSZJBW5O1ITB00M7AilxnNwDzqmou8Grg80lmtfeOBc6vqucA2wA/ausPAy6sqs2BC9trSZIkSZoSBm1N1pHApm2E+ZgkH0hydZKFST4OkGROkpuTfIVBUH5aknta+xuT/N82Un1JktuS7D5WZ1V1X1U91F6uA1TrYzbwUuBLrd0fqup3rd0ewKlt+VTgdZ3PgSRJkiSNyaCtyToM+EkbYf4usDmwAzAX2D7JS1u7zYHPVdWWVXUnsB5wUVVtCdwNfBL4M2BP4BPjdZjk+UluBBYBB7TgvQnwS+DkJAuSfDHJem2TJ1fVz9vyvwFPHmO/+yeZn2T+kvsWL8epkCRJkqQ/ZtDWitil/SwArgWewyBgA9xZVVeOaPsH4Py2vAj4XlU92JbnjNdJVV3VAvrzgA8lWQeYBWwHnFBV2wL3MsoU8aoq2ij4KO+dVFXzqmreWuvOHuJwJUmSJGliBm2tiABHVNXc9rNZVX2pvXfvMm0fbKEX4GHgAYCqephBaJ5QVf0IuAd4LnAXcFdVXdXePptB8Ab49yRPAWi/fzH5Q5MkSZKk5WPQ1mTdDWzQli8A3plkfYAkGyf5k56dJdlk6cPPkjyDwaj5HVX1b8C/JHl2a7oz8MO2fA7w9rb8duAfetYkSZIkSeMZaiRRWqqqfp3k8iQ3AOcBZwBXJIHBaPNbgSUdu3wJcFiSBxmMhB9UVb9q7x0MnJ7kscBtwL5t/ZHAN5LsB9wJvKljPZIkSZI0rjw6m1dacx14+BF13pKtx21zx5G7TVE1kiRJklZBGbahU8clSZIkSerIqeNaJSR5FXDUMqtvr6o9p6MeSZIkSVpeBm2tEqrqAgYPV5MkSZKkGc2p45IkSZIkdWTQliRJkiSpI4O2JEmSJEkdGbQlSZIkSerIoC1JkiRJUkcGbUmSJEmSOjJoS5IkSZLUkUFbkiRJkqSODNqSJEmSJHVk0JYkSZIkqaNZ012AtCrYauPZnHDQbtNdhiRJkqTVgCPakiRJkiR1ZNCWJEmSJKkjg7YkSZIkSR0ZtCVJkiRJ6sigLUmSJElSRwZtSZIkSZI6MmhLkiRJktSRQVuSJEmSpI4M2pIkSZIkdTRruguQVgWLfraYOYed+8jrO47cbRqrkSRJkjSTOaItSZIkSVJHBm1JkiRJkjoyaEuSJEmS1JFBW5IkSZKkjgzakiRJkiR1ZNCWJEmSJKkjg7YkSZIkSR0ZtCVJkiRJ6sigLUmSJElSRwZtSZIkSZI6MmhLkiRJktSRQVuSJEmSpI4M2pIkSZIkdWTQliRJkiSpI4O2JEmSJEkdGbQlSZIkSerIoK3ukhyT5MYkx6yEfX8xyRa99ytJkiRJvcya7gK0WtofeGJVLRmmcZJZVfXQMG2r6r+tUGWSJEmStJI5oq3/IMnbkixMcn2S05LMSXJRW3dhkqe3dqckOS7J95PcluSNbf05wPrANUn2nmD7E5NcBRyd5GNJTk1yWZI7k7w+ydFJFiU5P8nabbtLksxry/ck+VSr9cokT27rN22vFyX5ZJJ7puFUSpIkSVpDGbT1iCRbAlfUcPMAAA9ySURBVB8GXlFV2wDvBf4eOLWqtgZOB44bsclTgJcArwWOBKiq3YH7q2puVZ05wfZPBV5UVe9vrzcFXgHsDnwVuLiqtgLuB3YbpeT1gCtbrZcC72rrjwWObdveNc7x7p9kfpL5S+5bPPEJkiRJkqQhGLQ10iuAs6rqVwBV9RvghcAZ7f3TGATrpb5dVQ9X1Q+BJ4+xz/G2P2uZ6eXnVdWDwCJgLeD8tn4RMGeUff8B+Ke2fM2INi8EzmrLZzCGqjqpquZV1by11p09VjNJkiRJmhSDtlbEAyOWsxzb3zva/qrqYeDBqqq2/mFGf57AyDZLxmgjSZIkSVPKoK2RLgL2SvIkgCRPBL4P7NPefwtw2ST3uaLbL48rgTe05X3GayhJkiRJvTkCqEdU1Y1JPgV8L8kSYAFwMHBykg8AvwT2neRuV3T75XEI8NUkhzOYfu4N2JIkSZKmTB6deSutHpKsy+CBbJVkH+DNVbXHeNscePgRdd6SrR95fceRoz17TZIkSdIabOjbZR3R1upoe+CzSQL8DnjnNNcjSZIkaQ1i0NZqp6ouA7aZ7jokSZIkrZl8GJokSZIkSR0ZtCVJkiRJ6sigLUmSJElSRwZtSZIkSZI6MmhLkiRJktSRQVuSJEmSpI4M2pIkSZIkdWTQliRJkiSpI4O2JEmSJEkdGbQlSZIkSerIoC1JkiRJUkcGbUmSJEmSOpo13QVIq4KtNp7NCQftNt1lSJIkSVoNOKItSZIkSVJHBm1JkiRJkjoyaEuSJEmS1JFBW5IkSZKkjgzakiRJkiR1ZNCWJEmSJKkjg7YkSZIkSR0ZtCVJkiRJ6sigLUmSJElSRwZtCVj0s8XTXYIkSZKk1YRBW5IkSZKkjgzakiRJkiR1ZNCWJEmSJKkjg7YkSZIkSR0ZtCVJkiRJ6sigLUmSJElSRwZtSZIkSZI6MmhLkiRJktSRQVuSJEmSpI4M2pIkSZIkdWTQliRJkiSpI4O2JEmSJEkdGbQlSZIkSerIoC1JkiRJUkcGbUmSJEmSOjJoS5IkSZLU0aSDdpKPJTl0nPdPSfLGUdbPTbLrBPveKcmLJltT2/aAJG9bnm1XRUkOSbLuiNffSbLhSupr9ySHLcd23+/U/5wk9ye5bsTPY8dpv1OSf+rRtyRJkiT1NmsK+5oLzAO+M06bnYB7gEkHuKo6cfnKmh5JAqSqHh6jySHAV4H7AKpq3C8pVkRVnQOcsxzbLdeXImP4SVXN7bg/SZIkSZoWQ41oJzk8yS1J/hl4dlu3aZLzk1yT5LIkzxmxySuTzG/bvLaNTn4C2LuNVu49Sh9zgAOA97U2O7aRzouSLExyYZKnj1PjIyPtSS5JclSSH7Qadmzr10ryP5Pc0PZ5cFu/c5IFSRYl+XKSx7X1dyQ5otUzP8l2SS5I8pMkB4zo+wNJrm77/Pg4Nc5JcnOSrwA3AE9LckLb941Lt03yV8B/BS5OcvGIWjZqy+9vx3BDkkMm6O+mNsvgliSnJ3llksuT3Jpkh9buHUk+25b3avu9Psmlbd2W7Vxe145x87b+nvZ7p3bOz279nd6+SCDJrm3dNUmOm8xIdJIdklzRPpvvJ3n2KG1eNmIUfEGSDYb9TJLs3879/CX3LR62LEmSJEka14RBO8n2wD4MRqR3BZ7X3joJOLiqtgcOBT43YrM5wA7AbsCJrZ+PAGdW1dyqOnPZfqrqjtb2M63NZcDfA6dW1dbA6cBxkzi2WVW1A4OR4Y+2dfu32uYu3WeSdYBTgL2raisGo/wHjtjPT9tI62Wt3RuBFwBLQ/EuwObteOcC2yd56Th1bQ58rqq2rKo7gcOrah6wNfCyJFtX1XHAvwIvr6qXj9y4fR77As9vdbwrybbj9LcZ8HfAc9rPXwAvYfCZ/fUo7T8CvKqqtgF2b+sOAI5t52EecNco223L4FxvATwTeHE7t58HXtOuk/88Tp2bjgjMx7d1NwE7VtW2ra6/HWW7Q4H3tNp2BO4f9jOpqpOqal5VzVtr3dnjlCZJkiRJwxtm6viOwLeq6j6AJOcA6wAvAs5qA5cAjxuxzTfalOhbk9zGIOAtjxcCr2/LpwFHT2Lbb7bf1zAI1wCvBE6sqocAquo3SbYBbq+qW1qbU4H3AP+rvV46pXoRsH5V3Q3cneSBDO6Z3qX9LGjt1mcQ8i4do647q+rKEa/flGR/Bp/FUxgE1YXjHNdLGHwe9wIk+SaDz2jBGO1vr6pFre2NwIVVVUkWjTgvI10OnJLkGzx6Dq8ADk/yVOCbVXXrKNv9oKruav1c1/Z9D3BbVd3e2nyNwZcdoxlt6vhs4NQ2gl7A2mPU++kkp7fa7mpBezKfiSRJkiR1s7z3aD8G+N0499TWBK+nwgPt9xJW7F70pft5eMTy0tezgABHVNXnh9zfvUsXkmzCYET2eVX12ySnMPgSo6dlax55PH90XqrqgCTPZzAb4Zok21fVGUmuauu+k+TdVXXROP2s6Dlf6m+Ai6tqzwxuLbhklHqPTHIug9kWlyd5FZP/TCRJkiSpm2Hu0b4UeF2Sx7f7X/+cwQO6bk+yFwwe7NVGhpfaK8ljkmzKYBrxzcDdwAYT9LVsm+8zmLYO8BYG07dXxHeBdyeZ1ep+YqttTpLNWpu/BL43iX1eALwzyfptnxsn+ZMht30Cg+C9OMmTgdeMeG+s83UZg89j3STrAXuy4uflEUk2raqrquojwC8Z3Ef+TAYj08cB/8Bgmvswbgae2UIywB/dmz+B2cDP2vI7xql3UVUdBVzNYPbEinwmkiRJkrRCJgzaVXUtcCZwPXAegzADg+C7X5LrgRuBPUZs9lPgB639AVX1e+BiYIuM8TC05h+BPVubHYGDgX2TLGQQgN872QNcxhdbbQtb3X/RatuXwTT4RQxGeod+gnlV/R/gDOCKtv3ZTPyFwtJtr2cwvfmmto/LR7x9EnB+2sPQRmxzLYN7xX8AXAV8sarGmja+PI7J4KFwNzD4ouN64E3ADW1K+HOBrwyzo6q6HziIwXFcw+DLg8UASeYl+eIEuzgaOCLJAsYeIT+kPbxtIfAgcN6KfCaSJEmStKJSNR2zurWmSLJ+Vd3TnkJ+PHBrVX1muuta1oGHH1EnfOpD012GJEmSpFVXJm4yMNT/3ktaAe9qI+E3MpgK7n3TkiRJklZrPR5YNWlJ9uWPp4FfXlXvGWLbw4G9lll9VlV9qld9KyrJk4ALR3lr56r69UzvbzLa6PUqN4ItSZIkSSvLtATtqjoZOHk5t/0UsMqE6tG0cDvWE9lnfH+SJEmSpLE5dVySJEmSpI4M2pIkSZIkdWTQliRJkiSpI4O2JEmSJEkdGbQlSZIkSerIoC1JkiRJUkcGbUmSJEmSOjJoS5IkSZLUkUFbkiRJkqSODNqSJEmSJHVk0JaArTaePd0lSJIkSVpNGLQlSZIkSerIoC1JkiRJUkcGbUmSJEmSOjJoS5IkSZLUkUFbkiRJkqSODNqSJEmSJHVk0JYkSZIkqSODtiRJkiRJHRm0JUmSJEnqyKAtSZIkSVJHBm1JkiRJkjoyaEuSJEmS1JFBW5IkSZKkjgzakiRJkiR1ZNCWJEmSJKkjg7YkSZIkSR0ZtCVJkiRJ6sigLUmSJElSRwZtSZIkSZI6MmhLkiRJktSRQVuSJEmSpI4M2pIkSZIkdWTQliRJkiSpI4O2JEmSJEkdGbQlSZIkSerIoC1JkiRJUkcGbUmSJEmSOjJoS5IkSZLUkUFbkiRJkqSOUlXTXYM07T74wQ/evfbaa9883XVo9XHPPfdstP766/9quuvQ6sNrSr15Taknryf1topeU7/65Cc/+ephGhq0JSDJ/KqaN911aPXhNaXevKbUm9eUevJ6Um8z/Zpy6rgkSZIkSR0ZtCVJkiRJ6sigLQ2cNN0FaLXjNaXevKbUm9eUevJ6Um8z+pryHm1JkiRJkjpyRFuSJEmSpI4M2pIkSZIkdWTQ1holyauT3Jzkx0kOG+X9xyU5s71/VZI5U1+lZpIhrqn3J/lhkoVJLkzyjOmoUzPHRNfUiHZvSFJJZuz/+kQr3zDXU5I3tb9TNyY5Y6pr1MwyxL97T09ycZIF7d++XaejTs0MSb6c5BdJbhjj/SQ5rl1vC5NsN9U1Li+DttYYSdYCjgdeA2wBvDnJFss02w/4bVVtBnwGOGpqq9RMMuQ1tQCYV1VbA2cDR09tlZpJhrymSLIB8F7gqqmtUDPJMNdTks2BDwEvrqotgUOmvFDNGEP+jfow8I2q2hbYB/jc1FapGeYU4NXjvP8aYPP2sz9wwhTU1IVBW2uSHYAfV9VtVfUH4OvAHsu02QM4tS2fDeycJFNYo2aWCa+pqrq4qu5rL68EnjrFNWpmGebvFMDfMPgi8PdTWZxmnGGup3cBx1fVbwGq6hdTXKNmlmGuqQKe0JZnA/86hfVphqmqS4HfjNNkD+ArNXAlsGGSp0xNdSvGoK01ycbAv4x4fVdbN2qbqnoIWAw8aUqq00w0zDU10n7AeSu1Is10E15Tbdrc06rq3KksTDPSMH+jngU8K8nlSa5MMt7IkjTMNfUx4K1J7gK+Axw8NaVpNTXZ/9ZaZcya7gIkaU2Q5K3APOBl012LZq4kjwE+DbxjmkvR6mMWgymZOzGYcXNpkq2q6nfTWpVmsjcDp1TV3yV5IXBakudW1cPTXZg0lRzR1prkZ8DTRrx+als3apsksxhMefr1lFSnmWiYa4okrwQOB3avqgemqDbNTBNdUxsAzwUuSXIH8ALgHB+IpjEM8zfqLuCcqnqwqm4HbmEQvKXRDHNN7Qd8A6CqrgDWATaakuq0Ohrqv7VWRQZtrUmuBjZPskmSxzJ4QMc5y7Q5B3h7W34jcFFV1RTWqJllwmsqybbA5xmEbO991ETGvaaqanFVbVRVc6pqDoP7/nevqvnTU65WccP8u/dtBqPZJNmIwVTy26aySM0ow1xTPwV2BkjypwyC9i+ntEqtTs4B3taePv4CYHFV/Xy6ixqGU8e1xqiqh5L8d+ACYC3gy1V1Y5JPAPOr6hzgSwymOP2YwYMZ9pm+irWqG/KaOgZYHzirPVfvp1W1+7QVrVXakNeUNJQhr6cLgF2S/BBYAnygqpzJpVENeU39D+ALSd7H4MFo73DQQmNJ8jUGX/Zt1O7r/yiwNkBVncjgPv9dgR8D9wH7Tk+lkxeve0mSJEmS+nHquCRJkiRJHRm0JUmSJEnqyKAtSZIkSVJHBm1JkiRJkjoyaEuSJEmS1JFBW5IkSZKkjgzakiRJkiR19P8BwkuZpn0OgtwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1008x720 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# XGB var importance\n",
    "xgb_default.varimp_plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Shapley Importance for MXGB and XGB"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Shapey for MXGB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "First 5 rows of contributions:\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead>\n",
       "<tr><th style=\"text-align: right;\">  debt_to_income_ratio_missing.False</th><th style=\"text-align: right;\">  debt_to_income_ratio_missing.True</th><th style=\"text-align: right;\">  debt_to_income_ratio_missing.missing(NA)</th><th style=\"text-align: right;\">    term_360</th><th style=\"text-align: right;\">  conforming</th><th style=\"text-align: right;\">  loan_amount_std</th><th style=\"text-align: right;\">  loan_to_value_ratio_std</th><th style=\"text-align: right;\">  no_intro_rate_period_std</th><th style=\"text-align: right;\">  intro_rate_period_std</th><th style=\"text-align: right;\">  property_value_std</th><th style=\"text-align: right;\">  income_std</th><th style=\"text-align: right;\">  debt_to_income_ratio_std</th><th style=\"text-align: right;\">  BiasTerm</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td style=\"text-align: right;\">                        -2.98902e-05</td><td style=\"text-align: right;\">                                  0</td><td style=\"text-align: right;\">                                         0</td><td style=\"text-align: right;\">-2.06251e-05</td><td style=\"text-align: right;\"> -0.00374499</td><td style=\"text-align: right;\">        -0.780964</td><td style=\"text-align: right;\">               0.133854  </td><td style=\"text-align: right;\">                -0.0361677</td><td style=\"text-align: right;\">             -0.0434363</td><td style=\"text-align: right;\">           -0.123355</td><td style=\"text-align: right;\">   0.0101909</td><td style=\"text-align: right;\">                 -0.216221</td><td style=\"text-align: right;\">         0</td></tr>\n",
       "<tr><td style=\"text-align: right;\">                        -0.000281135</td><td style=\"text-align: right;\">                                  0</td><td style=\"text-align: right;\">                                         0</td><td style=\"text-align: right;\">-1.97926e-05</td><td style=\"text-align: right;\"> -0.00100632</td><td style=\"text-align: right;\">        -0.614443</td><td style=\"text-align: right;\">              -0.414059  </td><td style=\"text-align: right;\">                -0.0256902</td><td style=\"text-align: right;\">             -0.0618107</td><td style=\"text-align: right;\">           -0.428796</td><td style=\"text-align: right;\">  -0.0089271</td><td style=\"text-align: right;\">                 -0.937681</td><td style=\"text-align: right;\">         0</td></tr>\n",
       "<tr><td style=\"text-align: right;\">                        -8.52665e-05</td><td style=\"text-align: right;\">                                  0</td><td style=\"text-align: right;\">                                         0</td><td style=\"text-align: right;\">-1.77338e-05</td><td style=\"text-align: right;\"> -0.00730571</td><td style=\"text-align: right;\">        -0.465455</td><td style=\"text-align: right;\">              -0.00296585</td><td style=\"text-align: right;\">                -0.282846 </td><td style=\"text-align: right;\">             -0.572081 </td><td style=\"text-align: right;\">           -0.128206</td><td style=\"text-align: right;\">  -0.0700573</td><td style=\"text-align: right;\">                 -0.352035</td><td style=\"text-align: right;\">         0</td></tr>\n",
       "<tr><td style=\"text-align: right;\">                        -0.00103777 </td><td style=\"text-align: right;\">                                  0</td><td style=\"text-align: right;\">                                         0</td><td style=\"text-align: right;\">-3.13982e-05</td><td style=\"text-align: right;\"> -0.00894576</td><td style=\"text-align: right;\">        -1.82511 </td><td style=\"text-align: right;\">              -2.01086   </td><td style=\"text-align: right;\">                -0.143934 </td><td style=\"text-align: right;\">             -0.264119 </td><td style=\"text-align: right;\">           -0.676105</td><td style=\"text-align: right;\">  -0.123941 </td><td style=\"text-align: right;\">                 -0.850855</td><td style=\"text-align: right;\">         0</td></tr>\n",
       "<tr><td style=\"text-align: right;\">                        -0.000186064</td><td style=\"text-align: right;\">                                  0</td><td style=\"text-align: right;\">                                         0</td><td style=\"text-align: right;\">-2.06251e-05</td><td style=\"text-align: right;\"> -0.00368282</td><td style=\"text-align: right;\">        -0.728184</td><td style=\"text-align: right;\">               0.196378  </td><td style=\"text-align: right;\">                -0.0484225</td><td style=\"text-align: right;\">             -0.0394591</td><td style=\"text-align: right;\">           -0.230825</td><td style=\"text-align: right;\">  -0.0197131</td><td style=\"text-align: right;\">                 -0.23308 </td><td style=\"text-align: right;\">         0</td></tr>\n",
       "</tbody>\n",
       "</table>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "shap_values_mxgb_default, expected_value_mxgb_default = get_shapley(mxgb_default, train, X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAApwAAAFeCAYAAAAog8+HAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xm8X9O9//HXiaFKEmPQSiQxlKuDat8t2hqKm3sNqZb6xRihcVEp6iKKEKHSCI0qKoYKiaGm0oMoWtG4NfRDS6tUkYkmBIlm0JLk/P5Y68vO1xm+5zg75yR5Px+P8zjf7157r7X2/p488j5rrb1PXUNDA2ZmZmZmZenS0R0wMzMzsxWbA6eZmZmZlcqB08zMzMxK5cBpZmZmZqVy4DQzMzOzUjlwmpmZmVmpHDjNzMzMrFQOnGZmZmZWKgdOMzMzMyvVqh3dAbMVSX19fUP//v07uhtmZmbLSl0tO3mE08zMzMxK5cBpZmZmZqVy4DQzMzOzUjlwmpmZmVmpHDjNzMzMrFQOnGZmZmZWKgdOMzMzMyuVA6eZmZmZlcqB08zMzMxK5cBpZmZmZqVy4DQzMzOzUjlwmpmZmVmpHDjNzMzMrFQOnGZmZmZWKgdOMzMzMyuVA6eZmZmZlaquoaGho/tgtsKou3CR/0GZmVmn1HDyqmVUW1fLTh7hNDMzM7NSOXCamZmZWakcOM3MzMysVA6cZmZmZlYqB04zMzMzK5UDZyZpvqQdO7ofK4uPer0lXS1pXDt2qbm2XpQ0aFm0ZWZmtiIq5f745VFEdG3N/pIagJ0i4pGSutQmOYQtiojBHd2X5rT2epdB0iTgwYg4r6P7YmZmtiLzCGeJJK3WjnXVSVruf0Foz2tiZmZmy4flLsBImgpcCewObA9MBf4nIn6fA9npwCBgXeAp4ISI+EsN9b4/YpmnT88ELgFOBdYCbgG+GxGLJT2dD7tf0hLg5ogYnPv2c+DrwJeAwcDNko4FTgQ2Bp4DTomIyTX26UTgMODTwNclrQWcD3wKWAT8Bjg+Il6XdCpwSD72wFzN2rnP3wSGAZsDM4HzIuKGGvpQOad+wOeB54FjI+IPhX2OAk4AegEvA0Mj4v5cNhzYmfRZHJa/71k9Qixpf+AsoA/pMx0eEb8stHEkcAbQA7iL9KDZRTX0fw9gdD7vd4E/RcQeki4FdgJ2lHQa8GpEbJUD8SjgUGAJMKalNszMzKx5y+sI55HA8cDawAPAdXn7KcBAYC9SuJsMPCCpexva6A1sRAoqXwIOAA4EiIht8z79IqJr1fT1UcBJQDfgLkkHAefmfq0PXAXcJ6l3jf34DjAA6Ar8Efg3MIQUvD4LfBL4Se7XBcANwHW5X11z2PxP4BpSeF0POBy4VNLONfbhGFKgXA+4Dbi3ck1z2BxKCrrrkkLhHZK2KBy/Mynk9gL2r65c0ldyv08jXaPTgZskbZ/LdwIuy/1Yj/SZD6ix79eTfnFYG9gEOA8gIoaQfj7Ozddpq7z/acA+wFeAvqQAXOtnZWZmZo1Y7kY4s7ER8Sykm0eAEyWtDRwBjIqI53PZCNIo497ATa1s4x3grIhYDLwo6TeASMGoOVdFxB8rdUg6Ivf38bztGkmDgYOBkTX048KIeCm/XgwU14zOknQBaQSyOScAPymMqj4haQIpBP+uhj5cExFPAkgaBXyXFMpuzHWPiIjKqO+9kh4ihfPK2shpEXFRfv1uI/UPAm6PiIn5/T2Sfkn6xeLx3M/bIuKBXH69pKNr6Helvc2BjSJiFjCphf0HAj+KiBfz+Z5MCv1mZmbWRstr4JxZeL0gf+9GGkGbUimIiCV5SrhXG9p4PYfNYjvdajhuatX7XqTp+KKXWtGnpeqT9EXSlPq2wJqkqeWWbsDpS5qOP6mwbRXSCF+r+hARDZKmAz0LdV8m6ZLC/qsCrxTeT2uh/l7Ak1XbXgK+kF/3BKKqfAq12Zc0YvpnSbOBKyPi4mb278nS57tA0us1tmVmZmaNWF4DZ1NmkKZAAZDUJb+fUUJbDU1sX9Jcn7LNgPoa26mu72bStPYBEfFPSftU1VW9P6TANy4iRtfYZrU+lReS6oBN+SBQTgPOjohbmzm+sT4VNXWNKp/bq42U9wFebKFe8sjrgNzvr5HW3T4TEb9tol9LtZXXzPZoqR0zMzNr2ooWOMcBp0r6HWmUaijpHO8poa1ZwJYsPcXdVJ9+IulXfHDjzOeBg9rYbnfgbWCepE1Jaw6r+7WDpC4RUQlUFwPjJD0G/J40uvlZoC4iqkcOG3NknuL+M/B90shq5ZqOAYZL+jvwNLAG8EXgjcrShhpcBzwoaTzwIOkGpf2AXXP5eNK613HAw6Tp+u1pIXBKWp10ne+JiDckzSGFzMrI9Sxgi6rDxgOn5Ecm/QO4gOV3rbOZmVmnsKL9RzqatFbzfuA1YDfSjT3/LKGtM4ARkuZIGtvUThFxI3AOMAF4EzgW2CsiWppmbsr/kNalzgPuAKpHFq8m3VX/pqS5klbJd4wfRbo+b5CWJIyh5an4iitJN97MId2ss3dEvJ3P7ypSKLs2l08n3Q1f8+OPIuL/SDcyXZjruAA4NCIey+UPA9/L5/YW8N/AL2qsfgDwvKT5wK9Io7EP57IxgPJ1ejZvGwn8GniMNG0/nZaXBJiZmVkz6hoampoZNnv/sUhnRsSEju7L8qDuwkX+B2VmZp1Sw8mlTGzX1bLTijbCaWZmZmadzIq2hrNJecq0secpTouITy/r/gBImkh6+PiHLKs//SjpCtJDzhuzzbLow0ch6RCgqSUNR9fycHszMzMrl6fUzdqRp9TNzKyz6sgp9ZVmhNNsWfjVVhPp379/R3fDzMysU/EaTjMzMzMrlQOnmZmZmZXKgdPMzMzMSuXAaWZmZmalcuA0MzMzs1I5cJqZmZlZqRw4zczMzKxUfvC7WTvyg9/NrCOU9EBvs1r4b6mbmZmZWcdz4DQzMzOzUjlwmpmZmVmpHDjNzMzMrFQOnAWSJko6taP7sTL4qNda0qGSprZjl5pr62pJ45ZFW2ZmZisi39ZWEBF71rqvpEnAgxFxXnk9aj1Jg4AzI2KLju5Lc1pzrcsiaTjwtYjYo6P7YmZmtiLzCGeJJK3WmevrCCvCOZiZmVnreISzoDJqCUwApgADgR8AvYBHgcMjYqakS4GdgB0lnQa8GhFb5WnX1YD3gG8AvwCOlbQLcAGwNTATGBMRY2vsz5+APsBuwPmSJgBXA18EVgeeAU6MiCcl7QhcAawuaX6uZp+ImCTpM8BFwBeAd4AbgLMi4r0W+lA5pyXAvsBs4NyIGFfYZydgJLANMAe4HPhxRDRI2jVf0yOAc4AeQLfqEWJJnwMuBrbLdfwcGBkRi3P5l3O9W+drcn9L1y8f1wcYC2wPNJA+14OAzwOnA10K1+pzEfGypCOBM3Jf7yI9Y2xRLe2ZmZnZh3mEs3kDgJ2BTYC1gBEAETEEmEwKXl0jYqvCMQcAE0lh5X8l9QXuA34GrA8MAkZKOqDGPhwJXAKsnb93IQWv3sDGwFPAHZJWi4hHgWOAl3O/uuawuSHwMHBHPpcdgf8khela/D/g18B6wNHAzyR9BUDSNsC9wOh8znsDQ4DDCsevAuxFCpMbVVcuaW3gAeChfE575/M+qVA+Ebgt9+H7wHdr7Pv5wPTc7gak6z8nIn6RyyYVrtXLOTxfRrqO6+V+DaixLTMzM2uERzibd05EvAEg6UZgcA3HPJLDDMBCSQcBTxVGBB+TNDbXdWsN9d0WEb+t1EcKT9MrhZLOBI4HtgT+2kQdA4GnC6Oqr0oaCYwih+gWPBYRE/LrByXdTgpuvycFv1sj4q5c/nweAR4IXF+oY2hEvN1E/XsD7wLnRUQD8JykUaTAORrYB1gAjMrlf5B0DXBIDX1/lxRiN4uI50gjws0ZSLrmD+T310s6uoZ2zMzMrAkOnM2bWXi9AOhWwzFTq973Ik3jFr1Emp6uxVL1SdoA+DGwK7AOaaob0uhiU/oCX5U0t7CtjjTy2Oo+5PdfKNS9m6T9CuVdgBmF90uq3lfrBUzLYbLipbwdoGcj5dXXtCmnAMOAeklrkUZJfxAR85vYvycQVdtqbcvMzMwa4cDZdktq3D6DNJ1ctBnNB7Dm6hsJfALYPq8n7Qb8kw/+lmlj/ZpGWi+5d41tVuvTyPtXCnX/PCKOa+b4hqqwWG0G0FtSXWG/4jV6tZHy6j41KiJmk0aAj5e0GWlN5qnAWTR+rV5tpO4+wIu1tGdmZmYf5sDZdrOAWh49dBMwTNJA4EbSyODRwLFtbLc7aWp9jqSupGnx6n5tKKl7RPwzb7uetJ70yNyHd0kh6lMRcV8Nbe6QlwbcAuwC7A9UHiV0OfCwpPtIa1UbgE8BPSLi4RrP6R7SDUOnSxpNGjUdSrrZB+Bu0vrVUySNAT4LfAf4d0sVSxoAPEEalX2bdO6Lc/EsYFNJq0fEu3nbeOC+fLPUw8CBpBuOHDjNzMzayDcNtd0YQJLmSnq2qZ0iYgpphHMI8CYp0AyLiFva2O5ZwIa5rmdI6ygXF8ofIt3oMiX3bZeImAV8HfgmKXjNAX5JGkWsxS35HOYA1wDHRcT/5fP7C2mN5YmkJQivA+Nofop/KXltZz9SiH2NdIPS9aSlA0TEXNI6zwG5D5eQbsKqxXak4DgfeJZ0k9XoXHYraRR1Vr5WfXNI/h7pSQBvAf9NetqAmZmZtVFdQ0NzM522sssjfYsiopYbplZ6dRcu8j8oM1vmGk72hKV1mLqWd/EIp5mZmZmVzL8SdSBJp5MePt6YPSNi8jLowyF8sFayWqd/HJCkTWn6cVATIuKYZdkfMzMz+zBPqZu1I0+pm1lH8JS6daCaptT9E2rWjn611UT69+/f0d0wMzPrVLyG08zMzMxK5cBpZmZmZqVy4DQzMzOzUjlwmpmZmVmpHDjNzMzMrFQOnGZmZmZWKgdOMzMzMyuVH/xu1o784Hcrkx/ubWadkP+WupmZmZl1PAdOMzMzMyuVA6eZmZmZlcqB08zMzMxK5cBpZmZmZqVargKnpJ6SGiT1qWHf4ZIeXAbdahNJV0i6tKP70Zl1pmsk6UFJwzu6H2ZmZsujlfYZGzm0TgF6RcQrNew/CDgzIrZoj/Yj4pj2qGdFIWkq6fpOqGxbFtdI0jhgUUQMLrstMzOzldVyNcJpyx9Jq3V0H8zMzKxjdeoRTkkbA1cCuwCvARdUlR8FnAD0Al4GhkbE/YVd6iSNAQYC7wCXRsSPctnT+fvfJDUAoyLi3Cb6sSNwBbC6pPl58z4RMUnSLrlfWwMzgTERMbaGcxtHYWQt9+E44Ihc17PAoIh4PpevBpwCHA58Eng9n+9tklYFTgcGAesCTwEnRMRfCm2tArwH7AcsAE4GngOuyu0FcEhE/CMfsyYwAtgfWBt4AhgSES+2cF67Ag/m8zgH6AF0k3QCcCywCTAHuIE0orlYUj2wKXC1pCuA30dEv0auUW/gEuCrpM/zduAHEfFOC33aDvgp8FlgMfA8sDdwFHBI3ufAvPvawBLgNNLnsSZwHTU+2NbMzMw+rLOPcN5ACgibAjuTAhXwftgcSgoM6wJnAHdIKk5570wKqp8A9gVOknRwLts2f98qIro2FTYBIuJR4Bjg5bxv1xw2+wL3AT8D1s/9GynpgDae7yBSwNsAmEEKSRXnAYcCBwDdSSH8hVx2CilU7wVsDEwGHpDUvXD8t0kBbT3gXFLQHAF8C9gIaCAFxIpKEN0h1/k4cHeNI5ar5L5sl+sGeAXYM/d9X+BIYDBARPQHpgOD87XtV11hDtX3ALOA3rlfXwUurKE/lwH353PfCDgJeDciLiD9jF1X+FwXk67z93M/NwbeIP0smZmZWRt02hFOSZsAuwFbRMTbwNuSziEFB0gjmyMiojJSea+kh4ADSeEM0ojjqIhoAJ6UdCUp1N3YTt08CHgqIsbl949JGksKUre2ob7RETEd3h+VnJBf15FG2wZExDN531fyF6TRxFGF0dARuQ97AzflfX4bEffk8utJIXl8Zf2qpNtII35I2gA4GOgdEa/lbecAJwLbA4/UcC5D8+cGQETcXij7o6TxwO5Ai6PB2ZeBLYHtI2IBsEDSmcCdkobkz7gp75J+aekVEVOBx1poayAwNiKeBJA0kvQLh5mZmbVBpw2cQM/8fVph25TC677AZZIuKWxblQ9CGMC0qiAylTSl3F56VfUJ4CXSyFhbzCy8XgB0y697AGvxwYhms/2IiCX5JpxejdUdEQslVbe3sNBe3/z9mbxfxWpVdTZlCWmE9n2SDiKNLG5G+pxWp+XgV9QLmJ3DZsVLwBqk6/N6M8ceAQwDHpH0HinInxMRi5rYvyfpZwV4/3pOa2JfMzMza0FnDpyv5u+9ScECoE+hfBpwdkQ0N5LYW1JdIXT24YNAuqSV/Wls/xmkqeOizagKW+1gNikQbgn8vYl+9Km8kdQlv29rPyrhasuImN2G4xuKQV9SL1LI2w+YGBHvSroQKKbZlj6PGUAPSWtGxMK8bTPgX6Tr06SImEKawkfSZ0mj5FOAnzfR7qssfT3rSD+HZmZm1gadNnBGxCuSJgEXSDoC+DhwVmGXMcBwSX8n3QC0BvBF4I3K1DJp7eYp+cahz5CmjE/KZbNJYWNLlh4VbcosYENJ3SPin3nbTcAwSQNJ0/RfAI4m3RzTbiKiQdLlpGsxnXRD0SbAenmKfRxwqqTfkUbmhpI+23va2N7rkm4ELpd0YkS8Kmkd4OvAAxExv4UqqnUlrReeDbwnaQfgMNJNSxWzSJ9FU54AXgQukvS/wDqktajXtjCdjqTDc7//AcwFFpHWBlfa3UFSl4iohM/xpGv9S+DPpBusNq71ZM3MzGxpnf2moYOBj5FGtyYD11cKIuIq0t3h15Luep5OmjYt3tQymRQ6ZwF3Az8hr9/MdzYPA26SNFfSGS305SHgAWBK3n+XPHK2FzAEeJMUVIZFxC0f5aSbcAZwC3AnMA+YBFRukBpNCr/3k26S2g3oVwjGbXEU8DdgkqR5pOB1AOnmolaJiOeAs4G7SIHvND5YW1pxHnCopDmSJjZSxyJgH9J093RSAH2cFAZbshtpDe8C4FHSz8D4XHY1abnCm/lzXYX0c/ZToJ50PTcEflfzCZuZmdlS6hoaWp0fzKwJdRcu8j8oK03DyZ12UsrMVl41PTaws49wmpmZmdlyzr8uZ5J2Aj40lZudHxHnt7K+00kPY2/MnhExuTX1dSaFh99XmxwRey7TztD+n52ZmZm1L0+pm7Wj+vr6hv79+3d0N8zMzJYVT6mbmZmZWcdz4DQzMzOzUjlwmpmZmVmpHDjNzMzMrFQOnGZmZmZWKgdOMzMzMyuVA6eZmZmZlcrP4TRrRyvjn7b0n1s0M1up+TmcZmZmZtbxHDjNzMzMrFQOnGZmZmZWKgdOMzMzMyuVA6eZmZmZlcqB094naaKkUzu6H52NpKsljevofpiZmS2v/DwTe19E7NnRfViWJA0HvhYRe3R0X8zMzFZkHuE0MzMzs1J5hNPeJ2kS8CAwAZgCDAR+APQCHgUOj4iZed+uwHBgP6AHMAM4OiImS1oTGJnLPg48AhwfEdML7TwF9AX2AF4H/of08NiLgU2B3wADI2JePmZ94AKgH7AG8BDwvYh4rYVz6gOMBbYHGvJ5HQR8Hjgd6CJpft79cxHxsqQjgTPyed2V+7WoFZfSzMzMCjzCac0ZAOwMbAKsBYwolF1DCnG7A92BbwAzc9kYYIf81Rt4A6iXtErh+MOAHwHrAL8AxpNC585AH2Ar4HgASXXAnaTA+Jlc5zzgxhrO4XxgOrARsAEwCJgTEb/IZZMiomv+elnSTsBlwDHAesAD+TqYmZlZG3mE05pzTkS8ASDpRmBwfr0h8P+Az0TElLzvi7msC3A40D8iXs3bTgTeAr5MGikFuCUiHs/lE0gjqaMj4q287W5Aed8v5q89IuLfufxU4A1JPSPilWbO4V1gY2CziHgOeKaFcx4I3BYRD+T310s6uoVjzMzMrBke4bTmzCy8XgB0y6/75O8vNHJMD+BjpKlrACJiPmnavFcTdS9sYlulvb65ztckzZU0F3gJ+Bdp+r05p+S+1EuaKemneTlAU3oCU6u2TWlkPzMzM6uRRzitLabm71sCf60qmw38mxRKK6OeXYENSes822IaKfCuFxFLWnNgRMwmTc0fL2kz0prMU4GzgMbqepUPAnVFH/K5mJmZWes5cFqrRcTrkm4DLpc0iBQIN89lL0q6HjhX0l+BucBFwPPAE21tEngauETS2RHxpqQewO4RcXNzB0oakNudCrxNmmJfnItnAZtKWj0i3s3bxgP35eduPgwcSFqr6sBpZmbWRp5St7Y6EvgTKZTNI40cbpzLvk8KiX8g3bDzCeAbEbG4kXpalEc19yXdLf6kpHnAY8CuNRy+Xe7jfOBZ0t3xo3PZraRR11l5qr5vRDwMfA+4mrTu9L9JNzWZmZlZG9U1NDR0dB/MVhh1Fy5a6f5BNZzsiRIzs5VYXS07eYTTzMzMzErloQlbrknalA/fuFQxISKOWZb9MTMzsw/zlLpZO6qvr2/o379/R3fDzMxsWfGUupmZmZl1PAdOMzMzMyuVA6eZmZmZlcqB08zMzMxK5cBpZmZmZqVy4DQzMzOzUjlwmpmZmVmp/BxOs3ZU5p+29J+QNDOzTsjP4TQzMzOzjufAaWZmZmalcuA0MzMzs1I5cJqZmZlZqRw4zczMzKxUDpzWISTtKmlRR/ejVpIWSdq1o/thZma2PHLgXIlIGiTpxY7uR2ciaaqkQzu6H2ZmZisyB85lTNJqK1O7ZmZmZn6SdDuQNBX4OdAP+DzwPHBsRPxB0jhgNeA94BvAL4BjJe0CXABsDcwExkTE2FzfrsCDwHeAEUA34FfAkIiYn/dZPx/fD1gDeAj4XkS8VtWnrwNfAn4InA2sLml+7vo+wCjg5ogYUzifEcBXI2L3Zs55NLBlRHyzsG1XoB7YGGgAJgBfAdYEXgSGRsQDTdQ3DlgUEYOrruuZETEhv98JGAlsA8wBLgd+HBHNPmxd0oH53HsCC4H7IuJwSfXApsDVkq4Afh8R/SR1Ay4F+gPzgLOaq9/MzMya5xHO9nMMcAKwHnAbcK+k7rnsAGAi0AP4X0l9gfuAnwHrA4OAkZIOKNS3CinwfA74D+BTwI8BJNUBd5JC3WeA3qRgdGNVn44CTiIF1otzH1+OiK75axIwlhRsyXV3AY4ArmrhfK8F9pLUo7DtCOCWiFhA+tm6A9gyn+NNwO1V+9dM0jbAvcBo0nXcGxgCHNbCcWsC44HjIqIbsBlwNUBE9AemA4Pz9eiXD7s493sb0vXfl/R5mJmZWRt4hLP9XBMRTwJIGgV8lzSCCPBIRPwiv14o6SDgqYgYl7c9JmksMBi4tVDn0Ih4G3hb0lnA3ZKOAb4AfBHYIyL+nds8FXhDUs+IeCUff1VE/DG/fkdSY/2+GRgjaYeIeAz4L+DjwC+bO9mI+KukPwKH5uO7Ad8mjbiSR2InFA4ZLWkoabT13ubqbsJ3gVsj4q78/nlJlwIDgetbOPY9YGtJf4qIt4DJTe2YA/chwN4RMStvGwp8qw19NjMzMxw429PUyouIaJA0nTSFu1RZ1guYUrXtJdJIWtG0qvo/BmwA9M2vX6sKkf8iTRG/UjimWRGxUNIEUth9LH+/vhJkW3AtcCwwBvh/wCsR8X8Akj5OGo3cK/d5CWmktU0jnKRz3k3SfoVtXYAZzR2Uz28v0kjvDyW9DFwUEdWjwRU9SNd2amFb9WdlZmZmreDA2X76VF7kKe9K8NuGFLaKZpCCWNFmfDg89SYF0Ur9/wbeIAXRBcB6EVFdd1F1WVP7jgX+T9IPSdP4n2+mzqLK6OgXSMsCri2UnQTsDOwOTM0h/A2grom65pGCKQCSVgU2LJRPA34eEcfV2Lf35aUDkyStQlpHe7ukxyPiJT58Td4A3iVd7+K1NzMzszbyGs72c6SkL+S7wU8h3ShzTxP73gR8UdJASatK+jJwNHBN1X4jJXWXtCEwHBifA2YATwOX5JuHkNQj3xzTnFnAhoW1pQBExDPAs8DtwBMR8ddaTjgi5pKm3s8DdgCuKxR3JwXkN0k3Kp0FrNNMdU8Cu0vqK+ljpJucinfWXw4cKKm/pNXyddsm33zVJEkbSdpf0toRsRiYm4sW5++zSOs1K+e0mLQW9px8bHfgR821YWZmZs1z4Gw/VwKXkO6eHkBaA/h2YztGxBTSCOcQUiAbDwyLiFsKuy0mBdY/A38DXiaNGpJD576k0cInJc0jTYfv2kIfHwIeAKZImlsV1sYC29HyzULVrgX2BH4dETML239MCnf/II0ULqT5Kf4bSHfiP5X3nw68WimMiL+Q1sSeSLqr/3VgHC1P0XcBjgOm5ut0GXB4RFT6ch5wqKQ5kibmbSeQptGfJ13/ej4IqGZmZtZKdQ0NzT5RxmpQ/fiedqhvV+DBiFhmSx5ym3cCn4yIhcuq3RVN3YWLSvsH1XCyV8CYmVmn09RSuaV4hNOQtAZwMumudodNMzMza1ceMlnJ5bu+J5Cmsn9YVTYR2Kmx4yKia/m9q42k04HTmyjeMyKafAySmZmZlc9T6mbtqL6+vqF///4d3Q0zM7NlxVPqZmZmZtbxHDjNzMzMrFQOnGZmZmZWKgdOMzMzMyuVA6eZmZmZlcqB08zMzMxK5cBpZmZmZqXyczjN2lFjf9rSf5LSzMxWYH4Op5mZmZl1PAdOMzMzMyuVA6eZmZmZlcqB08zMzMxK5cBpZmZmZqVy4DQzMzOzUq2QgVPSJElndnQ/OhNJDZK+1tH9aA1Jz0oa0NH9AJC0SNKuHd0PMzOz5ZEfEFgSSYOAMyNii47uS2cnqQ8wBegVEa9UtkfEp5dB21NJn9OEstsyMzNbWa2QI5zWeUharaP7YGZmZh1rhR/hlPQ54GJgO2AO8HNgZEQszuXXAnsA6wAzgPMi4sZctivwIHAIcD6wAfBr4DsRMa+ZNncErgBWlzQ/b94nIiZJ2gW4ANgamAmMiYixLZzD3sC1wCYR8V7e1hWYBewdEQ9LOh84ENi1OW1pAAAgAElEQVQQeA34aURc3ER9g6gafZU0DlgUEYPz+02BHwNfAxqAeuB/mzvvfNxwYGfgKeCw/H3P5q4z8HT+/jdJDcCoiDi3evSxLdcuH3cgcDbQE1gI3BcRh0uqBzYFrpZ0BfD7iOgnqRtwKdAfmAec1VIbZmZm1rQVeoRT0trAA8BDwMbA3sCRwEmF3R4BPk8KQiOAcZK2KZSvAvQDtgU+RQquxzfXbkQ8ChwDvBwRXfPXJEl9gfuAnwHrA4OAkZIOaOFU7gMW5f5XHEAKnL/L7/9KCofdgKNyvf/VQr2NkrQG8NtcZ19gG1JY+0mNVexMCoS9gP3ztuau87b5+1b5Wp3bSJ/adO0krQmMB46LiG7AZsDVABHRH5gODM7t9suHXQxsmc/7c8C+pJ8DMzMza4MVfYRzb+Bd0mhaA/CcpFGkwDkaICKuKex/s6STgV1JYavitIiYD8yXdCegNvbnIOCpiBiX3z8maSwwGLi1qYMiYrGk8cARwJ158xHAtfm8qFqD+FtJ9wC7k0ZkW2sfoC4iKiN770gaBvxe0lGV0eFmTIuIi/Lrd3P/arnOzWnTtcveA7aW9KeIeAuY3NSOkrqQRrT3johZedtQ4Fs19tPMzMyqrOiBsxcp/DQUtr2Ut1fCxXBgAGkEtAFYC+hR2H9xRMwuvF9AGkVsa3+mVG17iTSC1pJrgWckbZjb/wpwcKVQ0vGkkc2eQB3wceDGRuqpRV9gU0lzq7Y3kK7Tqy0cP634psbr3JI2XbuIWChpL9IvGT+U9DJwUWE6v1oP4GPA1MK26nbNzMysFVboKXXSWsHekuoK2zbL2yGNmg0mTfuuGxHrkNYT1vHRLWmiP32qthX706SIeB54EjiUNJ38YOWObklfBUYBRwMb5POop+nzmEcKfEWfLLyeBrwQEetUfa0RES2FTfjwubd0nRu7VtU+yrWbFBHfIK3BPQ+YIGnzJtp+gzQqW2yrul0zMzNrhRV9hPMe0nq80yWNJo3cDQUqN5p0J62NnA10yTfTbAvc3Q5tzwI2lNQ9Iv6Zt90EDJM0kDT6+AVSSDy2xjqvJa0f7QacUtjeHVhMOo+GfJPRnjQ91fyn3Ld9gHtJo4Q7A5Vp+btJo4GnAz8F5pMC6Zcj4pc19rWopes8mxT8tgReaawC2njtJG1EWtv6YES8XRi1rSwLmJXbBd5fvnAjcI6kvwDvAD+q/VTNzMys2go9whkRb5Nu+NmDdOf2r4HrSXdfA1wHPA68SJom3oZm1ve10kOkG5amSJoraZeImALsBQwB3iTdzDIsIm6psc6bSaN6XYG7Ctsr5/UEaYTu20CTwTAiXgJOAK4E3gL+G7i9UL4Q2I10PZ4H3gZ+Q7rppy2avc4R8Q4wDLgpX6szGulzW69dF+A4YKqkecBlwOERMTWXnwccKmmOpIl52wmkafTngT+TRotbWrdqZmZmTahraGhoeS8zq0ndhYs+9A+q4eQVfSLBzMxWYjUtQ1yhRzjNzMzMrON56KWNJO0ETGyi+PyIOL+V9Z0OnN5E8Z4R0V5T/R9Je593e+hM1+5XW02kf//+y6o5MzOz5YKn1M3aUX19fYMDp5mZrUQ8pW5mZmZmHc+B08zMzMxK5cBpZmZmZqVy4DQzMzOzUjlwmpmZmVmpHDjNzMzMrFQOnGZmZmZWKgdOMzMzMyuVA6eZmZmZlcqB08zMzMxK5cBpZmZmZqVy4DQzMzOzUjlwmpmZmVmpHDjNzMzMrFSrdnQHbPklaRLwYESc19F9qZD0Q+BgYH3gX8DvgJMiYnphn82BC4Hd8qbngJ0i4r1cLuBy4DPATODsiJiwzE7CzMxsBeMRTuswklYrodrxwOcjojvQB5gO3FxoswcwGXga2BRYDxgCLM7lawMTgduBdYFjgCsk7VhCX83MzFYKHuG0NpF0KbATsKOk04BXI2IrSUcBJwC9gJeBoRFxfz5mOLAz8BRwGPCUpGOBKcAgYCjQG3gYOCS/PxJYApwbEZe11K+IeL7wti4fu1Vh20nA9IgYXjys8Ho/YCFwQUQ0AA9I+iXwP8CjLbVvZmZmH+YRTmuTiBhCGik8NyK6FsLmUFJYXBc4A7hD0haFQ3cmTVP3AvYvbN8f+Bpp1LEP8DjwEvBJ4AjgYkmb1tI3SQdLehuYTwq/wwvFXwdmSLpH0luSnpF0SKF8W+CPOWxWPJW3m5mZWRt4hNPa0wnAiIh4Or+/V9JDwIFAZZ3ntIi4KL9+Ny2XBFJwfQtA0t3A3hFxVS6bKGkOsB1pirxZEXEjcKOkjYHvAH8uFG8AfAkYAOxLCqD1kqZFxCNAN+DtqirnAt1bPHszMzNrlAOntae+wGWSLilsWxV4pfB+WhPHziy8Xlj1vrKtW2s6ExGzJF0FvCxp0xxo5wGPRsRtebcHJN0HfAN4JJf3qapqHeCfrWnbzMzMPuDAaR/Fkqr300h3dN/aimPKtiqwFmlq/i3gT8AWjexXmUJ/GvhmVdl2ebuZmZm1gQOnfRSzWDq8jQGGS/o7KaCtAXwReKPqZp5SSOoCfBe4JSJel9QT+CkwFai0PxaYLOmbwK+AXYB+wKhc/kvgAkmnAD8h3Ri1H/CfZfffzMxsReWbhuyjGEN6bOVcSc/mNZcXANcCc0jrLYcBZTz+qCl7AX+RtIB049FCYI+IWAQQEY+RntM5ijR9/lPg8Ih4NJfPzXUcQFrLeRVwTKXczMzMWq+uoaGh5b3MrCb19fUN/fv37+humJmZLSt1tezkEU4zMzMzK5XXcNpyRdIVwKFNFG9T/BOWZmZm1jk4cNpyJSKOIf25STMzM1tOeErdzMzMzErlwGlmZmZmpXLgNDMzM7NSOXCamZmZWakcOM3MzMysVA6cZmZmZlYqB04zMzMzK5UDp5mZmZmVyoHTzMzMzErlwGlmZmZmpXLgNDMzM7NSOXCamZmZWakcOM3MzMysVA6cHUjSJElndnQ/rGWSXpQ0qKP7YWZmtjxy4LTljqRdJS1qp7oc+s3MzErmwGlmZmZmpVq1oztgiaTPARcD2wFzgJ8DIyNicS6/FtgDWAeYAZwXETfmsl2BB4FDgPOBDYBfA9+JiHkttLstcAnwaWAV4DFgSES8lMvH5e3vAfsBC4CTgeeAq4CtgQAOiYh/5GPWB8YA/XIzvwa+HxFv5fKpwJkRMSG/7wNMAXpFxCuFNv8FHJDbHBERYyV9EpgIrCJpfq7/uIi4rplz3AMYDWwOvAv8KSL2kHQpsBOwo6TTgFcjYitJqwGjgEOBJflczMzMrI08wtkJSFobeAB4CNgY2Bs4EjipsNsjwOdJgXMEME7SNoXyVUgBb1vgU6TgenwNzTcAw4FNgD7AfGBC1T7fBm4H1gPOJQXNEcC3gI1yHecU9r8BWBf4j/y1ATC+hr5Ut1mf2/wecKmk3jnU7gksjoiu+avJsJldTwrVa+fzPA8gIoYAk4Fzcz1b5f1PA/YBvgL0JV2X3q3sv5mZmWUe4ewc9iaNvJ0XEQ3Ac5JGkQLnaICIuKaw/82STgZ2Bf5a2H5aRMwH5ku6E1BLDUfEM4W3/5Z0DvBnSWtGxMK8/bcRcQ+ApOuBnwHjI+KVvO024Kj8+pPAfwGfiog5edtJwPOSPhERM2u8Jr+NiF/l13dImksK3NNqPL7oXdLo5kYRMQuY1ML+A4EfRcSLAPlaf6cN7ZqZmRkOnJ1FL2BaDpsVL+XtSOpCGoUcQBoBbQDWAnoU9l8cEbML7xcA3VpqWNLmpFC7fd6/0ocefBDu3g+JEbFQ0lLbgIWFtnrl71OqzqVSVmvgrN6vpvNpwr7A6aQgPRu4MiIubmb/nsDUypuIWCDp9Ta2bWZmttLzlHrnMAPoLamusG2zvB3gIGAwsD+wbkSsAzwN1PHRXQHMAz4XEd2Br+btba270uc+hW2bVZXNIwXmik+2so0lrdk5Ip6OiAHAhsDRwEhJuzVT16sU+i+pOtybmZlZKzhwdg73AB8DTpe0uqStgKFAZRq9O7AImA10kXQkaa1me+hOGj2cK2kD0trMNstrLO8HLpK0jqR1gYuAiYXp9CeBgyR1ldQDGNbKZmaRbhrq29KO+XoeLmmDPII8hxQyFxfq2qLqsPHAKZI2l/Rx4AL8b8XMzKzN/J9oJxARb5Nu+NkDeI10V/f1wI/zLtcBjwMvkkbftiHd7NIevk+6U/ufuc6726HOQ0mjmH8DngfmktZFVpxJCnwzSespb25N5RHxAmkd6ROS5ko6rIVDBpDWkM4HfgWcHREP57IxgHI9z+ZtI0mfwWOkpQHTadvaUTMzMwPqGhoaWt7LzGpSX1/f0L9//47uhpmZ2bJS0xI8j3CamZmZWal8l/oKTtJOpAelN+b8iDh/WfanDJIOAcY2UXx0RNywLPtjZmZmS/OUulk78pS6mZmtZDylbmZmZmYdz4HTzMzMzErlwGlmZmZmpXLgNDMzM7NSOXCamZmZWakcOM3MzMysVA6cZmZmZlYqB04zMzMzK5UDp5mZmZmVyoHTzMzMzErlwGlmZmZmpXLgNDMzM7NStSlwSuopqUFSnxr2HS7pwba0syxIukLSpR3dj86sI69Re7Qt6RBJT7dXn8zMzKx1Vu3oDuTQOgXoFRGv1LD/IODMiNiiPdqPiGPao54VhaSppOs7obKtI69Re7QdETcAN7RDd8zMzKwNPKW+kpC0Wkf3wczMzFZONY1wStoYuBLYBXgNuKCq/CjgBKAX8DIwNCLuL+xSJ2kMMBB4B7g0In6UyypTnX+T1ACMiohzm+jHjsAVwOqS5ufN+0TEJEm75H5tDcwExkTE2BrObRywKCIG5/cNwHHAEbmuZ4FBEfF8Ll8NOAU4HPgk8Ho+39skrQqcDgwC1gWeAk6IiL8U2loFeA/YD1gAnAw8B1yV2wvgkIj4Rz5mTWAEsD+wNvAEMCQiXmzhvHYFHszncQ7QA+gm6QTgWGATYA5p5O/MiFgsqR7YFLha0hXA7yOiXyPXqDdwCfBV0ud5O/CDiHinhT4NB3bK53gk6ReeH+bjrwW+BLwAHBoRzxWu2aKIGCypDjgvn1M34E3gooj4qaR1ST+ju5F+rl8BjomIydWj4pImAU8CfYB+pM/wpIi4K5fXAT8AvgusCVwHfA6YHBHDmztHMzMz+7BaRzhvABaTwsjOpEAFvB82hwKHkELWGcAdkopT3juTguongH2BkyQdnMu2zd+3ioiuTYVNgIh4FDgGeDnv2zWHzb7AfcDPgPVz/0ZKOqDG86s2iBTwNgBmAD8tlJ0HHAocAHQnhfAXctkppFC9F7AxMBl4QFL3wvHfJgWs9YBzSUFzBPAtYCOggRQQKypBdIdc5+PA3TWOWK6S+7JdrhtSENsz931fUvAbDBAR/YHpwOB8bftVV5hD9T3ALKB37tdXgQtr6A+kn4W/53M5FBgNXEMK+euRwvclTRz7n6Sgv31EdAO+DDySy04hhcPewDqk69ncEo3DgYtIIf5S4Loc7gEOI/0C1Z903WbmfpuZmVkbtDjCKWkT0qjRFhHxNvC2pHOAygjmCcCIiKiMVN4r6SHgQFI4g/Qf9qiIaACelHQlKdTd2E7ncRDwVESMy+8fkzSWFKRubUN9oyNiOrw/wjYhv64jBaMBEfFM3vcVPgg2R5DOszIaOiL3YW/gprzPbyPinlx+PSkkj6+sX5V0G3BUfr0BcDDQOyJey9vOAU4EtueDsNWcoflzAyAibi+U/VHSeGB3oMXR4OzLwJak0LcAWCDpTOBOSUPyZ9ycFyLi6vx6oqQ3gV8XRjRvpOn1lu8CawCfljQ7Il4njU5WytYHtgL+GBEvNFFHxS8i4ve5zSuBH+fzepr0S8PYiPhjLh9N+tzNzMysDWoZ4eyZv08rbJtSeN0XuEzS3MoX8HXSlG3FtKogMrVQb3voVdUngJfy9raYWXi9gDR9C2laei0+GNFsth8RsYR0rsV+zCyUL2ykvYWF9vrm788Uru1bwGrUdm5LSCO075N0kKQ/SHpT0tukINWjhroqegGzc9iseIkUBGupZ2bV+4U0ff5LiYhJpCULZwKvS7pfknLxaOA3pOnv2ZKuk7RRY/VU96NwLpV2N6Hw855/dpe6jmZmZla7WtZwvpq/9yYFC0hr3yqmAWdHRHMjib0l1RVCZx8+GBVcUltX39fY/jNIU8dFm9H+IWE2KRBtSZoWbqwffSpvJHXJ79vaj0ro2TIiZrfh+IZi0JfUizRaux8wMSLelXQhoMIxLX0eM4AektYsBObNgH+Rrk+pIuJK4Mo8/T0cuAPYNIfGM4Az8prjCaQQOrANzbxK+nkH3h/ZbusvL2ZmZiu9FgNnRLySb7K4QNIRwMeBswq7jAGGS/o7aTpyDeCLwBuVqWXS2s1T8o1DnyFNGZ+Uy2aTQs6WNL/mrmIWsKGk7hHxz7ztJmCYpIGkafovAEeTbo5pNxHRIOly0rWYTrqhaBNgvTzFPg44VdLvSCObQ0nX+J42tvd6nmK+XNKJEfGqpHVII8gPRMT8Fqqo1pU0qj0beE/SDqT1is8V9plF+iya8gTwInCRpP8lrZc8F7i2hun0j0TSl4GP5T78G5hHWluMpP65Xy8A80kBeHEbmxoPjJJ0B+kzPp50g5iZmZm1Qa03DR1M+o9+BulGmOsrBRFxFenu8GtJdz1PB4aRpn0rJpNC5yzgbuAn5PWb+c7mYcBNedr4jBb68hDwADAl779LREwhjXAOId25PB4YFhG31Hh+rXEGcAtwJynwTAIqN0iNJoXf+0k3Se0G9CsE47Y4CvgbMEnSPODPpBuWWh3u8jrJs4G7gLnAaXywtrTiPOBQSXMkTWykjkXAPqQlEdNJ4e9x0t32ZetK+tl5g/Q59wMG5LLNgXrgn6Sw/w4p8LfF9cBlwL2kz7En8Bgp5JqZmVkr1TU0lDooZbbcy0sjZgCnRESzN7rV19c39O/ff9l0zMzMrOPV1bJTh/+lIbPOSNKBpFHsLqRncq4JfGjE18zMzFrW6QKnpJ1o+j/28yPi/FbWdzrpzubG7BkRk1tTX2dSePh9tckRsecy7Qzt/9l1sCF88KiovwB7RcScDuyPmZnZcstT6mbtyFPqZma2kqlpSt1/S93MzMzMSuXAaWZmZmalcuA0MzMzs1I5cJqZmZlZqRw4zczMzKxUDpxmZmZmVioHTjMzMzMrlQOnmZmZmZXKgdPMzMzMSuXAaWZmZmalcuA0MzMzs1I5cJqZmZlZqRw4zczMzKxUq3Z0B2zlIek04PvAWsDXI+IPy6jdK4BFETFkWbRnZmZmS3PgtGVCUk/gfOAzEfHXZdl2RByzLNszMzOzpXlK3ZaVPsCSjxI2Ja3Wft0xMzOzZcUjnLYUSV2B4cB+QA9gBnA08CQwMm//OPAIcHxETM/HTcr79AH6Aa8DJ0XEXZIGAOOAVSTNB16LiM0lrQ+MyfsD/Br4fkS8leucCvwc+DrwJWCwpK2BnYAAjiT90vRD4Hbg2rzfC8ChEfFcrmccaUp9cH7fABwHHAFsDTwLDIqI53N5N+AyYB9gHjAs92OPiJj0Ua6vmZnZysgjnFbtGmB7YHegO/ANYCYpGO6Qv3oDbwD1klYpHHs4cBGwNnApcJ2kNSPiF8CewOKI6BoRm+f9bwDWBf4jf20AjK/qz1HASUA34K68bWfg78DGwKHA6P/f3p2H21GUeRz/vmRhCwGURIEQbiIBdQAVCh4cAePCJqIoILIao0gcN+CR0YmgUVAWQUZmYFhNWJRFZgwTEMIiDMKMYAkoeyTkJhASEyCJCWFLqPmj6iR1O2e7J2e54f4+z3Oee3qvfru6+z3V3bdTub8GvA14Aji/xnqOAw5Jy3wW+Lds2M+B0cRkdCfgQGAAIiIi0hC1cMoqzrnhwOeI91nOSr2fds6tR0wmD/Lez03jngC8BOwO/F8a9zrv/f+m4ZcAPwPGAH8us6ytgP2A7b33i1K/k4AnnXNbeu/npVEv9d4/lL6/4pwDmOG9vyz1u8U59yIwPWvR/BUxma3mp1nr7BTg6vR9AHAUcID3fkHqNzHFRURERBqgFk7JdaW/Mwr9hwHrA6UkFO/9MuJl822y8eZlw19OXzepsKzSdLOyfjMLwwC6y0w7r9C9vNBveZXllpvHy9n4WwCDgdnZ8Py7iIiI9JISTsl1p79jCv0XAq+xOiEt3es5nHg5uhGl6bqyfqMLwwDebHD+jXoBeJ1420DJyDaXQURE5C1Fl9RlFe/9AufcDcCFzrlxxJa90v2WVwKnOeceBxYT79V8EnigwWU975y7DTjXOfcFwNI8b8kup7ed935luiQ/yTn3CPAq8aEkERERaZBaOKVoPPAw8D/EJ7RvJD6ccyLxyfA/AnOALYFPee9XrsWyjk7LeIqYvC4Gjl2L+TXLt4jrOAN4FLgdCMRWXhEREeklCyF0ugwifZpzbgdiQry19/75auNOmzYtHHTQQe0pmIiISOdZPSPpkrpIgXNuNLFV937iQ0TnAffUSjZFRESkPF1SF1nTBsAlwBLgEeJT70d2tEQiIiLrMLVwihSk12/u2OlyiIiIvFWohVNEREREWkoJp4iIiIi0lBJOEREREWkpJZwiIiIi0lJKOEVERESkpZRwioiIiEhLKeEUERERkZZSwikiIiIiLaWEU0RERERaSgmniIiIiLSUEk4RERERaSklnCIiIiLSUko4RURERKSllHCKiIiISEsp4RQRERGRllLCKSIiIiItpYRTRERERFpKCaeIiIiItJSFEDpdBpG3jPXXX//R119//dVOl6MvGzhw4BYrVqx4odPl6OsUp9oUo9oUo/ooTrVVidELIYT9a07fgjKJ9Fs77bTTq9571+ly9GXOOa8Y1aY41aYY1aYY1Udxqm1tY6RL6iIiIiLSUko4RURERKSllHCKNNclnS7AOkAxqo/iVJtiVJtiVB/Fqba1ipEeGhIRERGRllILp4iIiIi0lBJOEREREWkp/VskkV5yzm0PXAG8HXgRONZ7/9fCOAOA84H9gQCc6b2/rN1l7ZQ6Y3Qq8HlgJfAGMNF7P73dZe2keuKUjbsD8BBwoff+2+0rZWfVGyPn3OeAUwEj7nMf997/rZ1l7ZQ697fhwGRgG2AQcBfwTe/9ijYXtyOcc+cAhwBdwE7e+0fLjNOvj9tQd5waOnarhVOk9y4CLvDebw9cAFxcZpyjgO2AMcAHgUnOua62lbDz6onRA8Bu3vudgfHAdc65DdtYxr6gnjiVToQXA1PbWLa+omaMnHMOmATs473fEdgTWNLOQnZYPfVoIvBE2t92BnYFPtu+InbcVGBvYHaVcfr7cRvqi1NDx24lnCK9kFoJdgGuSb2uAXZxzg0rjHo4cKn3/k3v/ULiTnxY+0raOfXGyHs/3Xu/PHX+hdgy9fa2FbTDelGXAL4L3ATMaFPx+oRexOhE4Bzv/XwA7/0S732/eONXL2IUgE2cc+sB6wODgbltK2iHee/v9d4/W2O0fnvcLqknTo0eu5VwivTONsBc7/1KgPT3+dQ/N5KevxDnlBnnrareGOWOBWZ6759rQ/n6irri5Jx7H7AfcF7bS9h59dal9wKjnXP3OOcedM6d4pyzNpe1U+qN0WnA9sA8YD4w3Xt/XzsLug7oz8ftRtV97FbCKSId5Zz7MPFkeESny9LXOOcGEf/33YRSQiFlDSBeJt4H+DBwAHBMR0vU9xxGbI3aEtga2Ns5d2hniyTrst4eu5VwivTOs8DW6Z660r11W6X+uTnAtln3yDLjvFXVGyOccx8ErgYO9t4/1dZSdl49cdoSeBfwW+dcN3ACcJxzrr/8k+re7G83eO9f894vBW4Edm9rSTun3hh9A/hluly8hBijj7S1pH1ffz5u90ojx24lnCK94L1fADzM6l90RwAPpft9cr8mJgbrpXupDgZuaF9JO6feGDnndgOuAw713j/Y3lJ2Xj1x8t7P8d5v4b3v8t53Af9KvMfsK20vcAf0Yn/7FbCvc85Sq/DHgD+3r6Sd04sYzSI+fY1zbjDwcWCNJ5D7uX573O6NRo/dSjhFem8C8A3n3Axiq8EEAOfcb9PTsgBXAc8AfwX+APzIez+rE4XtkHpidCGwIXCxc+7h9NmpM8XtmHri1N/VE6NrgQXA48Tk6zHg8g6UtVPqidEJwF7OuUeIMZoBXNqJwnaCc+5859xzwAjgDufcY6m/jtuZOuPU0LFbr7YUERERkZZSC6eIiIiItJQSThERERFpKSWcIiIiItJSSjhFREREpKWUcIqIiIhISynhFOnHzGw/M/t91j3WzLo7WKS2MbMpZnZZE+fXZWYh6x5mZrPNbIs6pp1gZlc1qyzrAjPby8wWd7oc/ZGZHd2b/bzZ+4pU16p9o4HtfqaZndas5SvhFOmnzMyI7+f+QY3xvmpmj5rZ381skZl5Mzs8G95tZkeXmW6N/hbNSPMaUhg21syCmS1Ln+fNbLKZvW3t1rQzQggLif+QvFZ8NwZ+BExqQ7H6jBDC70MIm3W6HJWY2SQzu6PT5egPWhVrM7vbzE5p9nxbrbhvdLAungV8zcy2bsbMlHCK9F/7AoOBuyqNYGZHEBOmLwGbEl+ZdyKwqMFlfgQYDbxJ+ffvrgwhDAkhDAH2BD5IfLvOuuoXwBfNbGiVcY4GHgkhzGxTmXowswFmpnOBiPQQQlgE3AIc34z56SAj0gapte8UM7srtd49YmY7m9kRZva0mS0xs8vMbGA2zUgzu8HM5pvZPDO7xMw2yYb/xMyeSfObaWYnZMO6UmvhMWb2uJktNbPbzGzLrFgHA3eE6m9/+EfgnhDC/SF6Jf36vq3BUBwP3Ep8o0fVg1gI4RngJuADxWFmNjDF5OBC/ylmNjl9/5iZ3Z9aZRea2bVmNrzS8lK89sy6x5rZisIyJ6YW2sVmdp+ZVX0bUAjhr8ALxNcIVnIwcPUIcLMAAAvySURBVHuhLN8ysyfTdptjZmeY2YA07KdmNrUw/tg07sape0czm57WuzT9oDSsVDe+ZGaPA8uB4Wb2eTP7c2p9nmdmF5fml6Z7p5lNS3V1Rpo+mFlXNs5xqTV8iZk9ZGb7VlrpMvGdYmZXmdkvUnznpv3j/Wb2x7R+d5nZVtk03Wb2fTO7N+0H3sx2y4ZXrQNmNiht06fS/Gea2aEWW/AnAmNtdYv76Arr8eG0jCVpmx2fDRtrZivM7PA07yVmdn2+H5eZXyPHip3N7HdpPZ9J0w/Ihu+eYrPMzO4l/ujLl7mRmZ1jZrPM7CUzu9XMtqtUxjJlfruZXWnxWDXfzK6w7MqEFa52ZHVwRKVYm9m4tL7fSfVxgZmdW6Yej8jmO87Mnk7f/x3YCzg1zbPs+74tth7eaWZnpTryopmdZGbbppguNbM/mdl7smnWal/J6vqlWV1fo96k71XjU1iXHrc+NGm73048Rq29EII++ujT4g/QTXxd2nuAQcDVwEzgEmBjYCTx1XxHpfE3AJ4mXmrdENgc+C3wi2yeRxNbHA34KPAKsF8a1gUEYsK2BTAUuA+4NJv+fuCbhXKOBbqz7sOAV4HTie+n3qzCuh1dqz8wDHgN+CwxiQzAroVlr8i6twOeyte5MP+zgalZ9xBgGbBX6t4T2A0YCLwTuAe4Jht/CnBZ1h2APauU58cpZqOBAcRW3xeAzfOYlynnNOD0KnXjb8CnCv0OAUalbfuBNM7xadh7gdeBYdn4VwCXp+/DgReJCf1gYGvAA98v1I07U1wGp/U5APgHYkPEdsTXRJ6RLeNO4D9TXRoO3J3m05WGH0ess+9L8/hE2h7bVVjvYnynEOvwgWn6CWn6/ya+Zm8j4Hf0rMPdwPPArmk9vgssBIbWWQfOSuu5c4r1CGDnNGwS8QdZtf16VCrzuLSMPYCXgMOydQzE12wOAd5BPA58r4nHik1T/TgVWD9N9wxwcjb8xRSbwSke8+m5n/+SeKx4Rxrnh8CTwKBy+0qZMt9KrOebp8/NwM1VjgVdKS4jKsU6xfQN4ALiMfBdxNdxTiw3j2yap7Puu4FTamzDSWk5X2b1frASuKOwDW7PplnbfWUKsd58Ks3js6kM21bYNyrF5+lCv1XbqRnbPY2zK/GK1OBqcazn09aTrj769NdPOuCenHV/Ih2A8qTheuC89P1QYGZhHrsSE7YBFZZxA3B2+l46GO+WDf8a8FDWPQMYV5jH2PyAlPp9Evgv4kltJfES/I6FdXsZWFz4vEnPk8w/E0+UpZPYg8DFhWWHNO0iYBZwEWWS3DT+e4iJ1/DUPR6YUWUbfBJYkHWvOjin7ooJJzEZWQrsXZjnI6V1pHLC+Uvgwirleh0YW6P+nANcn3XfD5yYvm+S4v+h1P1t4HeF6Q8hnZyyurF3jWV+HXggfR+RphmdDf8YPU+ijwLHFuYxjQonfMonnHmSslGa/2FZv3+iZx3uBk7Lug2YAxxZqw6kcZcBB1YYdxK1E86JwH2FfmcA0wt1Ot/Pfwr8pso8u+ndseJI4FnSq6pTv+OBp9L3o1JM8uE/Ju3nxB+kARiZDV8PWELaH6iScBJ/9AZgTNZvh9Rvy2ydGkk4XwM2yvp9mbSPF+eRTdNIwvlYod+CMttgURP3lSlkdT31Wwh8usK+USk+1RLOtd7uqd+YNN7wanGs57OqSV5EWm5e9n058X7FhYV+pUtto4CRtuaTioHYUjPXzL5JbFUaQTx5bkh8SKXSMl/O5g8xqat2b2FcYAg3EX8FY2bvBi4EbjKzUSEdkYitb1fn01n2NKSZWSrr1SGEN1Lvy4EzzezbIYSlqd/KUOeDJCGEJ8zsQWJL78+ALwKTs2XuCvyE2OK2ETFGQ8rMqh5bpGmnWfYkOrH1Y0T5SVYZSkyeK1ljO1i8d/YkYmvqQGLrwx+yUSYDXyU+9PU54LkQwn1p2CjgQ4W6Y8TWm1x3YZn7AN8H3k1sKRtAPPFCbCWFeAIrmV2Y3yjgAjM7P+s3EHiO+q2qryGE5bHarLHfFC9Hd2fTBDObQ9omNerAMGKL4YxelK9oG9bctjOBT2fdxf28uB+W05tjxTbA7GxfLJVhm/R9RJnheZlHpb9/SfEuGZTNo5rSOPk8Z2bD5tG4BSGE5Vl3N7X3t0YUy7icKvWuCftKuWXWUy96o1nbfSirGwLWiu7hFOmbZhN/yW9W+GwQQphrZh8iXg48HtgiJWnTiCfUej1EvDxbtxDCk8QkZ1vipbN6fZR46Wl86T4v4uWbIcQWmkZNBsal+472AK7Mhl1LbEXdPoQwlPIPKeWWEROQkq2y7y8QTwgfL2yPjUMIZ9aY747EWFfSYzuY2TbES3inE1uINiVeVsy37bXA9ma2C7GlY3I2bDaxNSQv56YhPoiVezNb5mBgaprvyBSv72TLnJv+jsymz7+Xlju+sNwhIYSvVln3ZugqfUk/bEayOsmtVgcWEhOJMRXm+2aF/rln8+Uno1P/dnkW2NZ6Zg15GeaWGd6VfS8lQ2MK226jEMI1dS6/OM/RhWFLqbxvQeVYDzezjQrlLm3b0o/URubbsCbtK71Vbj2KMYWe69+s7b4jsQX49QbLvooSTpG+6SZgsMUHGjaxaGsz+0waPpR4eXshEMzsQOJ9Rb0xlXippyIzG29mh1n6X5LpBv0JwOMhhJd6sazjiffPvRt4f/rsSEyUvtLLcueuJSay5xPvsZqbDRtKvDy01MxGEu9lquZPwBfMbHC6uf+k0oDUSvBz4BwzGwNgZkMs/h/T4klulZQIDyPeD1bJVHo+VDSEeGxeCLxhZnsAx+QThBAWA78hJqV7EO/hLLkScGnbbWBm66WHDPavUobBxJaaRSGEV8zsvcTLhKXlPUe8PHlmqo/DgOK/mzkPmGTxIR8zsw3NbM/UKt5K481sF4sPk5xMbMm8OQ2rWAfSNr0QONviQ1Zm8SGWndMo84lXGQZXWfY1wK5mdqzFh8p2J9b1y5u6htXdTNx2E1Pd3YGYAJXKcBOxTp1s8SGpXYj3HwMQQlhAvDJyoaV/f2Nmm5nZZ6zwr8vKCSE8D9wGnJum2xw4F7glhFBqxfsTcETaZ4YR7zfNVYr1esBZqS6NJt4uckVa7oukHzkW/9PCTsSrKMX51v3wU52asa/0Vrn4PExMyD+Z9vHPAHtnw5u13fchHqPWmhJOkT4oXUb6KLHl60niSfNOYqIGMJ2YWDxAbH07lJiA9MZ0YIWZja0yziLipdsnzOxl4r2Di4n3wtXF4lPBBwPnhBDm5x9iK+0HrMbT3pWEEJYQ1/sA4r8gyn2FeM/XUuI9qL+uMbuvE09OLxHvkZtSGP4D4EbgRjP7O/HBjglUP46OB6akclZyFfC+dEIlhPBEtqzFxCSpXEvTZOJ6T89O7KS4foQY827iNvwNhSdUcyGEZcTtfLaZLSO2qBZvzziSmMw9R3wArRTP19I8LiU+yDU5LXMOMbEYVGXdm+ES4g+ORcDhxHsyS/GuVQe+R9zWU9M4d7M6Qfk1sYVuvsUniUcVpiWEMIt4f9/XiQ9oXAWcGkK4vlkrV0ta132JP1r+xupjw8/S8MXEB7EOJ8bofOA/CrM5jviA3t1mtpR4b/JhxEup9TiaGL+niMerxcCx2fBTiD+Q5xFjfG1h+kqxnk2sb7OIx55biXWs5AvEY9GStL7FRP884o+vxWb2WJ3rUlUz9pUGrBGfEP+N2reI9f8lYH/ig0qlcq71djezzYj1+6IGy92D9by8LyL9SWr1mhhC2Dt1jyUmSF2dLNe6KLWKzgohWOoeRnw63BXuvys37QTiQz/HVBuvLzGz/YhJ8YahQycSi/cJn1K8f1jWfWY2jrhtm91C2XZ9YV9phJmdQbx/uCn/PF8PDYn0YyGEW4mtBtJkKcncts5xL6JJrQitYmbvJ95L9gjxgYPTgevWpROoSDu8VfaVEMK/NHN+uqQuIrlu1u03+3TSYuKDUG9VmxMvSy8D7gX+QrykJyI9aV8pQ5fURURERKSl1MIpIiIiIi2lhFNEREREWkoJp4iIiIi0lBJOEREREWkpJZwiIiIi0lL/D6S0bMoFmE9JAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x396 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "shap.summary_plot(shap_values_mxgb_default, X, plot_type=\"bar\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Shapley for XGB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "First 5 rows of contributions:\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead>\n",
       "<tr><th style=\"text-align: right;\">  debt_to_income_ratio_missing.False</th><th style=\"text-align: right;\">  debt_to_income_ratio_missing.True</th><th style=\"text-align: right;\">  debt_to_income_ratio_missing.missing(NA)</th><th style=\"text-align: right;\">    term_360</th><th style=\"text-align: right;\">  conforming</th><th style=\"text-align: right;\">  loan_amount_std</th><th style=\"text-align: right;\">  loan_to_value_ratio_std</th><th style=\"text-align: right;\">  no_intro_rate_period_std</th><th style=\"text-align: right;\">  intro_rate_period_std</th><th style=\"text-align: right;\">  property_value_std</th><th style=\"text-align: right;\">  income_std</th><th style=\"text-align: right;\">  debt_to_income_ratio_std</th><th style=\"text-align: right;\">  BiasTerm</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td style=\"text-align: right;\">                        -4.35354e-05</td><td style=\"text-align: right;\">                                  0</td><td style=\"text-align: right;\">                                         0</td><td style=\"text-align: right;\"> 0.000756169</td><td style=\"text-align: right;\">-0.000440774</td><td style=\"text-align: right;\">        -0.363497</td><td style=\"text-align: right;\">                0.0929364</td><td style=\"text-align: right;\">                -0.0271633</td><td style=\"text-align: right;\">             -0.0102367</td><td style=\"text-align: right;\">           -0.355676</td><td style=\"text-align: right;\">   -0.558442</td><td style=\"text-align: right;\">                 -0.344734</td><td style=\"text-align: right;\">         0</td></tr>\n",
       "<tr><td style=\"text-align: right;\">                        -0.00026653 </td><td style=\"text-align: right;\">                                  0</td><td style=\"text-align: right;\">                                         0</td><td style=\"text-align: right;\"> 0.0138007  </td><td style=\"text-align: right;\"> 0.00216415 </td><td style=\"text-align: right;\">        -0.282659</td><td style=\"text-align: right;\">               -0.776825 </td><td style=\"text-align: right;\">                -0.0346796</td><td style=\"text-align: right;\">             -0.0789705</td><td style=\"text-align: right;\">           -0.983352</td><td style=\"text-align: right;\">   -0.516417</td><td style=\"text-align: right;\">                 -0.919282</td><td style=\"text-align: right;\">         0</td></tr>\n",
       "<tr><td style=\"text-align: right;\">                        -0.00011001 </td><td style=\"text-align: right;\">                                  0</td><td style=\"text-align: right;\">                                         0</td><td style=\"text-align: right;\">-0.00522695 </td><td style=\"text-align: right;\"> 0.00174915 </td><td style=\"text-align: right;\">        -0.237179</td><td style=\"text-align: right;\">               -0.283227 </td><td style=\"text-align: right;\">                -0.312884 </td><td style=\"text-align: right;\">             -0.818544 </td><td style=\"text-align: right;\">           -0.22195 </td><td style=\"text-align: right;\">   -0.408122</td><td style=\"text-align: right;\">                 -0.572551</td><td style=\"text-align: right;\">         0</td></tr>\n",
       "<tr><td style=\"text-align: right;\">                        -0.000589084</td><td style=\"text-align: right;\">                                  0</td><td style=\"text-align: right;\">                                         0</td><td style=\"text-align: right;\"> 0.00655149 </td><td style=\"text-align: right;\"> 0.00419639 </td><td style=\"text-align: right;\">        -0.519215</td><td style=\"text-align: right;\">               -1.63001  </td><td style=\"text-align: right;\">                -0.206296 </td><td style=\"text-align: right;\">             -0.113357 </td><td style=\"text-align: right;\">           -1.18688 </td><td style=\"text-align: right;\">   -0.226466</td><td style=\"text-align: right;\">                 -0.796942</td><td style=\"text-align: right;\">         0</td></tr>\n",
       "<tr><td style=\"text-align: right;\">                         0.000121103</td><td style=\"text-align: right;\">                                  0</td><td style=\"text-align: right;\">                                         0</td><td style=\"text-align: right;\"> 0.000828575</td><td style=\"text-align: right;\">-0.000766235</td><td style=\"text-align: right;\">        -0.327656</td><td style=\"text-align: right;\">               -0.0205416</td><td style=\"text-align: right;\">                -0.0259274</td><td style=\"text-align: right;\">             -0.0134669</td><td style=\"text-align: right;\">           -0.236151</td><td style=\"text-align: right;\">   -0.291408</td><td style=\"text-align: right;\">                 -0.179639</td><td style=\"text-align: right;\">         0</td></tr>\n",
       "</tbody>\n",
       "</table>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "shap_values_xgb_default, expected_value_xgb_default = get_shapley(xgb_default, train, X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAApwAAAFeCAYAAAAog8+HAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3XucX9O9//HXxKVKEtegJZK4lKMX1b5btHUpTs5xmWqpX1wjNA4qRR1EESJUGqFRRcWlQuJSt9JBFK1onLr0Q0urVJEbTQgSzUVLkvn9sdaX7Wsu3xmzM5Pk/Xw85jHf7157r7X2/k54z1pr76lrbGzEzMzMzKws3Tq7A2ZmZma2fHPgNDMzM7NSOXCamZmZWakcOM3MzMysVA6cZmZmZlYqB04zMzMzK5UDp5mZmZmVyoHTzMzMzErlwGlmZmZmpVq5sztgtjxpaGhorK+v7+xumJmZLS11tezkEU4zMzMzK5UDp5mZmZmVyoHTzMzMzErlwGlmZmZmpXLgNDMzM7NSOXCamZmZWakcOM3MzMysVA6cZmZmZlYqB04zMzMzK5UDp5mZmZmVyoHTzMzMzErlwGlmZmZmpXLgNDMzM7NSOXCamZmZWakcOM3MzMysVA6cZmZmZlaqusbGxs7ug9lyo+6CRf4HZWZmXVLjSSuXUW1dLTt5hNPMzMzMSuXAaWZmZmalcuA0MzMzs1I5cJqZmZlZqRw4zczMzKxUDpwFkiZKOqWz+7Ei+KjXWtIhkqZ2YJdaausqSeOWRltmZmbLo1Luj19WRcQete4raRLwQEScW16P2k7SIOCMiNi8s/vSkrZc67JIGg58LSJ27+y+mJmZLc88wlkiSat05fo6w/JwDmZmZtY2HuEsqIxaAhOAKcBA4AdAb+AR4LCImCnpEmBHYAdJpwKvRMSWedp1FeBd4BvAL4BjJO0MnA9sBcwExkTE2Br78yegL7ArcJ6kCcBVwBeBVYGngRMi4glJOwCXA6tKmp+r2TsiJkn6DHAh8AXgbeB64MyIeLeVPlTOaQmwDzAbOCcixhX22REYCWwNzAEuA34cEY2SdsnX9HDgbKAX0KN6hFjS54CLgG1zHT8HRkbE4lz+5VzvVvma3Nfa9cvH9QXGAtsBjaTP9UDg88BpQLfCtfpcRLwk6Qjg9NzXO0kPtV1US3tmZmb2YR7hbNkAYCdgI2ANYARARAwBJpOCV/eI2LJwzP7ARFJY+V9J/YB7gZ8B6wKDgJGS9q+xD0cAFwNr5u/dSMGrD7Ah8CRwu6RVIuIR4Gjgpdyv7jlsrg88BNyez2UH4D9JYboW/w/4NbAOcBTwM0lfAZC0NXAPMDqf817AEODQwvErAXuSwuQG1ZVLWhO4H3gwn9Ne+bxPLJRPBG7Nffg+8N0a+34eMD23ux7p+s+JiF/kskmFa/VSDs+Xkq7jOrlfA2psy8zMzJrgEc6WnR0RrwNIugEYXMMxD+cwA7BQ0oHAk4URwUcljc113VJDfbdGxG8r9ZHC0/RKoaQzgOOALYC/NlPHQOCpwqjqK5JGAqPIIboVj0bEhPz6AUm3kYLb70nB75aIuDOXP5dHgAcC1xXqGBoRbzVT/17AO8C5EdEIPCtpFClwjgb2BhYAo3L5HyRdDRxcQ9/fIYXYTSPiWdKIcEsGkq75/fn9dZKOqqEdMzMza4YDZ8tmFl4vAHrUcMzUqve9SdO4RS+Spqdr8YH6JK0H/BjYBViLNNUNaXSxOf2Ar0qaW9hWRxp5bHMf8vsvFOreVdK+hfJuwIzC+yVV76v1BqblMFnxYt4OsHET5dXXtDknA8OABklrkEZJfxAR85vZf2MgqrbV2paZmZk1wYGz/ZbUuH0GaTq5aFNaDmAt1TcS+ASwXV5P2gP4JylANtevaaT1knvV2Ga1vk28f7lQ988j4tgWjm+sCovVZgB9JNUV9iteo1eaKK/uU5MiYjZpBPg4SZuS1mSeApxJ09fqlSbq7gu8UEt7ZmZm9mEOnO03C6jl0UM3AsMkDQRuII0MHgUc0852e5Km1udI6k6aFq/u1/qSekbEP/O260jrSY/IfXiHFKI+FRH31tDm9nlpwM3AzsB+QOVRQpcBD0m6l7RWtRH4FNArIh6q8ZzuJt0wdJqk0aRR06Gkm30A7iKtXz1Z0hjgs8B3gH+3VrGkAcDjpFHZt0jnvjgXzwI2kbRqRLyTt40H7s03Sz0EHEC64ciB08zMrJ1801D7jQEkaa6kZ5rbKSKmkEY4hwBvkALNsIi4uZ3tngmsn+t6mrSOcnGh/EHSjS5Tct92johZwNeBb5KC1xzgl6RRxFrcnM9hDnA1cGxE/F8+v7+Q1lieQFqC8Bowjpan+D8gr+3sTwqxr5JuULqOtHSAiJhLWuc5IPfhYtJNWLXYlhQc5wPPkG6yGp3LbiGNos7K16pfDsnfIz0J4E3gv0lPGzAzM7N2qmtsbGmm01Z0eaRvUUTUcsPUCq/ugkX+B2VmZl1S40mlTGzXtb6LRzjNzMzMrGRew9mJJJ1Gevh4U/aIiMlLoQ8H8/5ayWpd/nFAkjah+cdBTYiIo5dmf8zMzOzDPKVu1oE8pW5mZl1VZ06pe4TTrAP9asuJ1NfXd3Y3zMzMuhSv4TQzMzOzUjlwmpmZmVmpHDjNzMzMrFQOnGZmZmZWKgdOMzMzMyuVA6eZmZmZlcqB08zMzMxK5Qe/m3UgP/jdzGzFVNJD1ZcF/lvqZmZmZtb5HDjNzMzMrFQOnGZmZmZWKgdOMzMzMyuVA6eZmZmZlWqZCpySNpbUKKlvDfsOl/TAUuhWu0i6XNIlnd2PrqwrXSNJD0ga3tn9MDMzWxatsPfw59A6BegdES/XsP8g4IyI2Lwj2o+IozuinuWFpKmk6zuhsm1pXCNJ44BFETG47LbMzMxWVMvUCKcteySt0tl9MDMzs87VpUc4JW0IXAHsDLwKnF9VfiRwPNAbeAkYGhH3FXapkzQGGAi8DVwSET/KZU/l73+T1AiMiohzmunHDsDlwKqS5ufNe0fEJEk7535tBcwExkTE2BrObRyFkbXch2OBw3NdzwCDIuK5XL4KcDJwGPBJ4LV8vrdKWhk4DRgErA08CRwfEX8ptLUS8C6wL7AAOAl4FrgytxfAwRHxj3zM6sAIYD9gTeBxYEhEvNDKee0CPJDP42ygF9BD0vHAMcBGwBzgetKI5mJJDcAmwFWSLgd+HxH9m7hGfYCLga+SPs/bgB9ExNut9Glb4KfAZ4HFwHPAXsCRwMF5nwPy7msCS4BTSZ/H6sC11PhgWzMzM/uwrj7CeT0pIGwC7EQKVMB7YXMoKTCsDZwO3C6pOOW9EymofgLYBzhR0kG5bJv8fcuI6N5c2ASIiEeAo4GX8r7dc9jsB9wL/AxYN/dvpKT923m+g0gBbz1gBikkVZwLHALsD/QkhfDnc9nJpFC9J7AhMBm4X1LPwvHfJgW0dYBzSEFzBPAtYAOgkRQQKypBdPtc52PAXTWOWK6U+7JtrhvgZWCP3Pd9gCOAwQARUQ9MBwbna9u/usIcqu8GZgF9cr++ClxQQ38uBe7L574BcCLwTkScT/oZu7bwuS4mXefv535uCLxO+lkyMzOzduiyI5ySNgJ2BTaPiLeAtySdTQoOkEY2R0REZaTyHkkPAgeQwhmkEcdREdEIPCHpClKou6GDunkg8GREjMvvH5U0lhSkbmlHfaMjYjq8Nyo5Ib+uI422DYiIp/O+L+cvSKOJowqjoSNyH/YCbsz7/DYi7s7l15FC8vjK+lVJt5JG/JC0HnAQ0CciXs3bzgZOALYDHq7hXIbmzw2AiLitUPZHSeOB3YBWR4OzLwNbANtFxAJggaQzgDskDcmfcXPeIf3S0jsipgKPttLWQGBsRDwBIGkk6RcOMzMza4cuGziBjfP3aYVtUwqv+wGXSrq4sG1l3g9hANOqgshU0pRyR+ld1SeAF0kjY+0xs/B6AdAjv+4FrMH7I5ot9iMiluSbcHo3VXdELJRU3d7CQnv98ven834Vq1TV2ZwlpBHa90g6kDSyuCnpc1qV1oNfUW9gdg6bFS8Cq5Guz2stHHs4MAx4WNK7pCB/dkQsamb/jUk/K8B713NaM/uamZlZK7py4Hwlf+9DChYAfQvl04CzIqKlkcQ+kuoKobMv7wfSJW3sT1P7zyBNHRdtSlXY6gCzSYFwC+DvzfSjb+WNpG75fXv7UQlXW0TE7HYc31gM+pJ6k0LevsDEiHhH0gVAMc229nnMAHpJWj0iFuZtmwL/Il2fZkXEFNIUPpI+SxolnwL8vJl2X+GD17OO9HNoZmZm7dBlA2dEvCxpEnC+pMOBjwNnFnYZAwyX9HfSDUCrAV8EXq9MLZPWbp6cbxz6DGnK+MRcNpsUNrbgg6OizZkFrC+pZ0T8M2+7ERgmaSBpmv4LwFGkm2M6TEQ0SrqMdC2mk24o2ghYJ0+xjwNOkfQ70sjcUNJne3c723tN0g3AZZJOiIhXJK0FfB24PyLmt1JFte6k9cKzgXclbQ8cSrppqWIW6bNozuPAC8CFkv4XWIu0FvWaVqbTkXRY7vc/gLnAItLa4Eq720vqFhGV8DmedK1/CfyZdIPVhrWerJmZmX1QV79p6CDgY6TRrcnAdZWCiLiSdHf4NaS7nqeTpk2LN7VMJoXOWcBdwE/I6zfznc3DgBslzZV0eit9eRC4H5iS9985j5ztCQwB3iAFlWERcfNHOelmnA7cDNwBzAMmAZUbpEaTwu99pJukdgX6F4JxexwJ/A2YJGkeKXjtT7q5qE0i4lngLOBOUuA7lffXllacCxwiaY6kiU3UsQjYmzTdPZ0UQB8jhcHW7Epaw7sAeIT0MzA+l11FWq7wRv5cVyL9nP0UaCBdz/WB39V8wmZmZvYBdY2Nbc4PZtaMugsW+R+UmdkKqPGkLjtpXLaaHhvY1Uc4zczMzGwZt8LG8WqSdgQ+NJWbnRcR57WxvtNID2Nvyh4RMbkt9XUlhYffV5scEXss1c7Q8Z+dmZmZdSxPqZt1oIaGhsb6+vrO7oaZmdnS4il1MzMzM+t8DpxmZmZmVioHTjMzMzMrlQOnmZmZmZXKgdPMzMzMSuXAaWZmZmalcuA0MzMzs1L5OZxmHch/2tLsw1bgP/lntiLwczjNzMzMrPM5cJqZmZlZqRw4zczMzKxUDpxmZmZmVioHzkzSfEk7dHY/VhQf9XpLukrSuA7sUkttvSBp0NJoy8zMbHnkWweziOjelv0lNQI7RsTDJXWpXXIIWxQRgzu7Ly1p6/Uug6RJwAMRcW5n98XMzGx55hHOEklapQPrqpO0zP+C0JHXxMzMzJYNy1yAkTQVuALYDdgOmAr8T0T8Pgey04BBwNrAk8DxEfGXGup9b8QyT5+eAVwMnAKsAdwMfDciFkt6Kh92n6QlwE0RMTj37efA14EvAYOBmyQdA5wAbAg8C5wcEZNr7NMJwKHAp4GvS1oDOA/4FLAI+A1wXES8JukU4OB87AG5mjVzn78JDAM2A2YC50bE9TX0oXJO/YHPA88Bx0TEHwr7HAkcD/QGXgKGRsR9uWw4sBPpszg0f9+jeoRY0n7AmUBf0mc6PCJ+WWjjCOB0oBdwJ+m5X4tq6P/uwOh83u8Af4qI3SVdAuwI7CDpVOCViNgyB+JRwCHAEmBMa22YmZlZy5bVEc4jgOOANYH7gWvz9pOBgcCepHA3GbhfUs92tNEH2IAUVL4E7A8cABAR2+R9+kdE96rp6yOBE4EewJ2SDgTOyf1aF7gSuFdSnxr78R1gANAd+CPwb2AIKXh9Fvgk8JPcr/OB64Frc7+657D5n8DVpPC6DnAYcImknWrsw9GkQLkOcCtwT+Wa5rA5lBR01yaFwtslbV44fidSyO0N7FdduaSv5H6fSrpGpwE3Stoul+8IXJr7sQ7pMx9QY9+vI/3isCawEXAuQEQMIf18nJOv05Z5/1OBvYGvAP1IAbjWz8rMzMyasMyNcGZjI+IZSDePACdIWhM4HBgVEc/lshGkUca9gBvb2MbbwJkRsRh4QdJvAJGCUUuujIg/VuqQdHju72N529WSBgMHASNr6McFEfFifr0YKK4ZnSXpfNIIZEuOB35SGFV9XNIEUgj+XQ19uDoingCQNAr4LimU3ZDrHhERlVHfeyQ9SArnlbWR0yLiwvz6nSbqHwTcFhET8/u7Jf2S9IvFY7mft0bE/bn8OklH1dDvSnubARtExCxgUiv7DwR+FBEv5PM9iRT6zczMrJ2W1cA5s/B6Qf7egzSCNqVSEBFL8pRw73a08VoOm8V2etRw3NSq971J0/FFL7ahTx+oT9IXSVPq2wCrk6aWW7sBpx9pOv7EwraVSCN8bepDRDRKmg5sXKj7UkkXF/ZfGXi58H5aK/X3Bp6o2vYi8IX8emMgqsqnUJt9SCOmf5Y0G7giIi5qYf+N+eD5LpD0Wo1tmZmZWROW1cDZnBmkKVAAJHXL72eU0FZzfzN7SUt9yjYFGmpsp7q+m0jT2vtHxD8l7V1VV/X+kALfuIgYXWOb1fpWXkiqAzbh/UA5DTgrIm5p4fim+lTU3DWqfG6vNFHeF3ihlXrJI68Dcr+/Rlp3+3RE/LaZfn2grbxmtldr7ZiZmVnzlrfAOQ44RdLvSKNUQ0nneHcJbc0CtuCDU9zN9eknkn7F+zfOfB44sJ3t9gTeAuZJ2oS05rC6X9tL6hYRlUB1ETBO0qPA70mjm58F6iKieuSwKUfkKe4/A98njaxWrukYYLikvwNPAasBXwReryxtqMG1wAOSxgMPkG5Q2hfYJZePJ617HQc8RJqu345WAqekVUnX+e6IeF3SHFLIrIxczwI2rzpsPHByfmTSP4DzWXbXOpuZmXUJy9v/SEeT1mreB7wK7Eq6seefJbR1OjBC0hxJY5vbKSJuAM4GJgBvAMcAe0ZEa9PMzfkf0rrUecDtQPXI4lWku+rfkDRX0kr5jvEjSdfnddKShDG0PhVfcQXpxps5pJt19oqIt/L5XUkKZdfk8umku+FrfvxRRPwf6UamC3Id5wOHRMSjufwh4Hv53N4E/hv4RY3VDwCekzQf+BVpNPahXDYGUL5Oz+RtI4FfA4+Spu2n0/qSADMzM2tBXWNjczPDZu89FumMiJjQ2X1ZFtRdsMj/oMyqNJ60vE2mmVlBXS07LW8jnGZmZmbWxawwv3bmKdOmnqc4LSI+vbT7AyBpIunh4x+ytP70o6TLSQ85b8rWS6MPH4Wkg4HmljQcVcvD7c3MzKxcnlI360CeUjf7ME+pmy3XappS938FzDrQr7acSH19fWd3w8zMrEvxGk4zMzMzK5UDp5mZmZmVyoHTzMzMzErlwGlmZmZmpXLgNDMzM7NSOXCamZmZWakcOM3MzMysVH7wu1kHWp4f/O6Hd5uZWRP8t9TNzMzMrPM5cJqZmZlZqRw4zczMzKxUDpxmZmZmVioHTjMzMzMrlQOnvUfSREmndHY/uhpJV0ka19n9MDMzW1b5OSf2nojYo7P7sDRJGg58LSJ27+y+mJmZLc88wmlmZmZmpfIIp71H0iTgAWACMAUYCPwA6A08AhwWETPzvt2B4cC+QC9gBnBUREyWtDowMpd9HHgYOC4iphfaeRLoB+wOvAb8D+nhsRcBmwC/AQZGxLx8zLrA+UB/YDXgQeB7EfFqK+fUFxgLbAc05vM6EPg8cBrQTdL8vPvnIuIlSUcAp+fzujP3a1EbLqWZmZkVeITTWjIA2AnYCFgDGFEou5oU4nYDegLfAGbmsjHA9vmrD/A60CBppcLxhwI/AtYCfgGMJ4XOnYC+wJbAcQCS6oA7SIHxM7nOecANNZzDecB0YANgPWAQMCcifpHLJkVE9/z1kqQdgUuBo4F1gPvzdTAzM7N28ginteTsiHgdQNINwOD8en3g/wGfiYgped8Xclk34DCgPiJeydtOAN4EvkwaKQW4OSIey+UTSCOpoyPizbztLkB53y/mr90j4t+5/BTgdUkbR8TLLZzDO8CGwKYR8SzwdCvnPBC4NSLuz++vk3RUK8eYmZlZCzzCaS2ZWXi9AOiRX/fN359v4phewMdIU9cARMR80rR572bqXtjMtkp7/XKdr0qaK2ku8CLwL9L0e0tOzn1pkDRT0k/zcoDmbAxMrdo2pYn9zMzMrEYe4bT2mJq/bwH8tapsNvBvUiitjHp2B9YnrfNsj2mkwLtORCxpy4ERMZs0NX+cpE1JazJPAc4EmqrrFd4P1BV9yediZmZmbefAaW0WEa9JuhW4TNIgUiDcLJe9IOk64BxJfwXmAhcCzwGPt7dJ4CngYklnRcQbknoBu0XETS0dKGlAbncq8BZpin1xLp4FbCJp1Yh4J28bD9ybn7v5EHAAaa2qA6eZmVk7eUrd2usI4E+kUDaPNHK4YS77Pikk/oF0w84ngG9ExOIm6mlVHtXch3S3+BOS5gGPArvUcPi2uY/zgWdId8ePzmW3kEZdZ+Wp+n4R8RDwPeAq0rrT/ybd1GRmZmbtVNfY2NjZfTBbbtRdsGi5/QfVeJInRMzM7EPqatnJI5xmZmZmVioPWdgyTdImfPjGpYoJEXH00uyPmZmZfZin1M06UENDQ2N9fX1nd8PMzGxp8ZS6mZmZmXU+B04zMzMzK5UDp5mZmZmVyoHTzMzMzErlwGlmZmZmpXLgNDMzM7NSOXCamZmZWan8HE6zDtTV/7Sl/zylmZl1MD+H08zMzMw6nwOnmZmZmZXKgdPMzMzMSuXAaWZmZmalcuA0MzMzs1I5cFqnkLSLpEWd3Y9aSVokaZfO7oeZmdmyyIFzBSJpkKQXOrsfXYmkqZIO6ex+mJmZLc8cOJcySausSO2amZmZ+SnQHUDSVODnQH/g88BzwDER8QdJ44BVgHeBbwC/AI6RtDNwPrAVMBMYExFjc327AA8A3wFGAD2AXwFDImJ+3mfdfHx/YDXgQeB7EfFqVZ++DnwJ+CFwFrCqpPm563sDo4CbImJM4XxGAF+NiN1aOOfRwBYR8c3Ctl2ABmBDoBGYAHwFWB14ARgaEfc3U984YFFEDK66rmdExIT8fkdgJLA1MAe4DPhxRLT4sHVJB+Rz3xhYCNwbEYdJagA2Aa6SdDnw+4joL6kHcAlQD8wDzmypfjMzM2uZRzg7ztHA8cA6wK3APZJ65rL9gYlAL+B/JfUD7gV+BqwLDAJGStq/UN9KpMDzOeA/gE8BPwaQVAfcQQp1nwH6kILRDVV9OhI4kRRYL8p9fCkiuuevScBYUrAl190NOBy4spXzvQbYU1KvwrbDgZsjYgHpZ+t2YIt8jjcCt1XtXzNJWwP3AKNJ13EvYAhwaCvHrQ6MB46NiB7ApsBVABFRD0wHBufr0T8fdlHu99ak678P6fMwMzOzdvAIZ8e5OiKeAJA0CvguaQQR4OGI+EV+vVDSgcCTETEub3tU0lhgMHBLoc6hEfEW8JakM4G7JB0NfAH4IrB7RPw7t3kK8LqkjSPi5Xz8lRHxx/z6bUlN9fsmYIyk7SPiUeC/gI8Dv2zpZCPir5L+CBySj+8BfJs04koeiZ1QOGS0pKGk0dZ7Wqq7Gd8FbomIO/P75yRdAgwErmvl2HeBrST9KSLeBCY3t2MO3AcDe0XErLxtKPCtdvTZzMzMcODsSFMrLyKiUdJ00hTuB8qy3sCUqm0vkkbSiqZV1f8xYD2gX379alWI/BdpivjlwjEtioiFkiaQwu6j+ft1lSDbimuAY4AxwP8DXo6I/wOQ9HHSaOSeuc9LSCOt7RrhJJ3zrpL2LWzrBsxo6aB8fnuSRnp/KOkl4MKIqB4NruhFurZTC9uqPyszMzNrAwfOjtO38iJPeVeC39aksFU0gxTEijblw+GpDymIVur/N/A6KYguANaJiOq6i6rLmtt3LPB/kn5Imsb/fAt1FlVGR79AWhZwTaHsRGAnYDdgag7hrwN1zdQ1jxRMAZC0MrB+oXwa8POIOLbGvr0nLx2YJGkl0jra2yQ9FhEv8uFr8jrwDul6F6+9mZmZtZPXcHacIyR9Id8NfjLpRpm7m9n3RuCLkgZKWlnSl4GjgKur9hspqaek9YHhwPgcMAN4Crg43zyEpF755piWzALWL6wtBSAingaeAW4DHo+Iv9ZywhExlzT1fi6wPXBtobgnKSC/QbpR6UxgrRaqewLYTVI/SR8j3eRUvLP+MuAASfWSVsnXbet881WzJG0gaT9Ja0bEYmBuLlqcv88irdesnNNi0lrYs/OxPYEftdSGmZmZtcyBs+NcAVxMunt6AGkN4FtN7RgRU0gjnENIgWw8MCwibi7stpgUWP8M/A14iTRqSA6d+5BGC5+QNI80Hb5LK318ELgfmCJpblVYGwtsS+s3C1W7BtgD+HVEzCxs/zEp3P2DNFK4kJan+K8n3Yn/ZN5/OvBKpTAi/kJaE3sC6a7+14BxtD5F3w04Fpiar9OlwGERUenLucAhkuZImpi3HU+aRn+OdP0beD+gmpmZWRvVNTa2+EQZq0H143s6oL5dgAciYqktecht3gF8MiIWLq12lzd1Fyzq0v+gGk/yKhozM+tQzS2V+wCPcBqSVgNOIt3V7rBpZmZmHcrDHSu4fNf3BNJU9g+ryiYCOzZ1XER0L793tZF0GnBaM8V7RESzj0EyMzOz8nlK3awDNTQ0NNbX13d2N8zMzJYWT6mbmZmZWedz4DQzMzOzUjlwmpmZmVmpHDjNzMzMrFQOnGZmZmZWKgdOMzMzMyuVA6eZmZmZlcrP4TTrQB/lT1v6z06amdkyyM/hNDMzM7PO58BpZmZmZqVy4DQzMzOzUjlwmpmZmVmpHDjNzMzMrFQOnGZmZmZWKgfOTiRpkqQzOrsf1jpJL0ga1Nn9MDMzWxY5cNoyR9IukhZ1UF0O/WZmZiVz4DQzMzOzUvlPm3QRkj4HXARsC8wBfg6MjIjFufwaYHdgLWAGcG5E3JDLdgEeAA4GzgPWA34NfCci5rXS7jbAxcCngZWAR4EhEfFiLh+Xt78L7AssAE4CngWuBLYCAjg4Iv6Rj1kXGAP0z838Gvh+RLyZy6cCZ0TEhPy+LzAF6B0RLxfa/Bewf25zRESMlfRJYCKwkqT5uf5jI+LaFs5xd2A0sBkxEZajAAAgAElEQVTwDvCniNhd0iXAjsAOkk4FXomILSWtAowCDgGW5HMxMzOzdvIIZxcgaU3gfuBBYENgL+AI4MTCbg8DnycFzhHAOElbF8pXIgW8bYBPkYLrcTU03wgMBzYC+gLzgQlV+3wbuA1YBziHFDRHAN8CNsh1nF3Y/3pgbeA/8td6wPga+lLdZkNu83vAJZL65FC7B7A4Irrnr2bDZnYdKVSvmc/zXICIGAJMBs7J9WyZ9z8V2Bv4CtCPdF36tLH/ZmZmlnmEs2vYizTydm5ENALPShpFCpyjASLi6sL+N0k6CdgF+Gth+6kRMR+YL+kOQK01HBFPF97+W9LZwJ8lrR4RC/P230bE3QCSrgN+BoyPiJfztluBI/PrTwL/BXwqIubkbScCz0n6RETMrPGa/DYifpVf3y5pLilwT6vx+KJ3SKObG0TELGBSK/sPBH4UES8A5Gv9nXa0a2ZmZjhwdhW9gWk5bFa8mLcjqRtpFHIAaQS0EVgD6FXYf3FEzC68XwD0aK1hSZuRQu12ef9KH3rxfrh7LyRGxEJJH9gGLCy01Tt/n1J1LpWyWgNn9X41nU8z9gFOIwXp2cAVEXFRC/tvDEytvImIBZJea2fbZmZmKzxPqXcNM4A+kuoK2zbN2wEOBAYD+wFrR8RawFNAHR/d5cA84HMR0RP4at7e3rorfe5b2LZpVdk8UmCu+GQb21jSlp0j4qmIGACsDxwFjJS0awt1vUKh/5Kqw72ZmZm1gQNn13A38DHgNEmrStoSGApUptF7AouA2UA3SUeQ1mp2hJ6k0cO5ktYjrc1st7zG8j7gQklrSVobuBCYWJhOfwI4UFJ3Sb2AYW1sZhbppqF+re2Yr+dhktbLI8hzSCFzcaGuzasOGw+cLGkzSR8Hzsf/VszMzNrN/xPtAiLiLdINP7sDr5Lu6r4O+HHe5VrgMeAF0ujb1qSbXTrC90l3av8z13lXB9R5CGkU82/Ac8Bc0rrIijNIgW8maT3lTW2pPCKeJ60jfVzSXEmHtnLIANIa0vnAr4CzIuKhXDYGUK7nmbxtJOkzeJS0NGA67Vs7amZmZkBdY2Nj63uZWU3qLljU7n9QjSd5SbWZmS1zalqC5xFOMzMzMyuVh1SWc5J2JD0ovSnnRcR5S7M/ZZB0MDC2meKjIuL6pdWXX205kfr6+qXVnJmZ2TLBU+pmHaihoaHRgdPMzFYgnlI3MzMzs87nwGlmZmZmpXLgNDMzM7NSOXCamZmZWakcOM3MzMysVA6cZmZmZlYqB04zMzMzK5Wfw2nWgZr605b+k5VmZrYc83M4zczMzKzzOXCamZmZWakcOM3MzMysVA6cZmZmZlYqB04zMzMzK9VyGTglTZJ0Rmf3oyuR1Cjpa53dj7aQ9IykAZ3dDwBJiyTt0tn9MDMzWxb5eS0lkTQIOCMiNu/svnR1kvoCU4DeEfFyZXtEfHoptD2V9DlNKLstMzOzFdVyOcJpXYekVTq7D2ZmZta5lvsRTkmfAy4CtgXmAD8HRkbE4lx+DbA7sBYwAzg3Im7IZbsADwAHA+cB6wG/Br4TEfNaaHMH4HJgVUnz8+a9I2KSpJ2B84GtgJnAmIgY28o57AVcA2wUEe/mbd2BWcBeEfGQpPOAA4D1gVeBn0bERc3UN4iq0VdJ44BFETE4v98E+DHwNaARaAD+t6XzzscNB3YCngQOzd/3aOk6A0/l73+T1AiMiohzqkcf23Pt8nEHAGcBGwMLgXsj4jBJDcAmwFWSLgd+HxH9JfUALgHqgXnAma21YWZmZs1brkc4Ja0J3A88CGwI7AUcAZxY2O1h4POkIDQCGCdp60L5SkB/YBvgU6TgelxL7UbEI8DRwEsR0T1/TZLUD7gX+BmwLjAIGClp/1ZO5V5gUe5/xf6kwPm7/P6vpHDYAzgy1/tfrdTbJEmrAb/NdfYDtiaFtZ/UWMVOpEDYG9gvb2vpOm+Tv2+Zr9U5TfSpXddO0urAeODYiOgBbApcBRAR9cB0YHBut38+7CJgi3zenwP2If0cmJmZWTss7yOcewHvkEbTGoFnJY0iBc7RABFxdWH/mySdBOxCClsVp0bEfGC+pDsAtbM/BwJPRsS4/P5RSWOBwcAtzR0UEYsljQcOB+7Imw8HrsnnRdUaxN9KuhvYjTQi21Z7A3URURnZe1vSMOD3ko6sjA63YFpEXJhfv5P7V8t1bkm7rl32LrCVpD9FxJvA5OZ2lNSNNKK9V0TMytuGAt+qsZ9mZmZWZXkPnL1J4af4961fzNsr4WI4MIA0AtoIrAH0Kuy/OCJmF94vII0itrc/U6q2vUgaQWvNNcDTktbP7X8FOKhSKOk40sjmxqS/a/px4IYm6qlFP2ATSXOrtjeSrtMrrRw/rfimxuvcmnZdu4hYKGlP0i8ZP5T0EnBhYTq/Wi/gY8DUwrbqds3MzKwNluspddJawT6Sin9YftO8HdKo2WDStO/aEbEWaT1hTX+IvhVLmulP36ptxf40KyKeA54ADiFNJz9QuaNb0leBUcBRwHr5PBpo/jzmkQJf0ScLr6cBz0fEWlVfq0VEa2ETPnzurV3npq5VtY9y7SZFxDdIa3DPBSZI2qyZtl8njcoW26pu18zMzNpgeR/hvJu0Hu80SaNJI3dDgcqNJj1JayNnA93yzTTbAHd1QNuzgPUl9YyIf+ZtNwLDJA0kjT5+gRQSj6mxzmtI60d7ACcXtvcEFpPOozHfZLQHzU81/yn3bW/gHtIo4U5AZVr+LtJo4GnAT4H5pED65Yj4ZY19LWrtOs8mBb8tgJebqoB2XjtJG5DWtj4QEW8VRm0rywJm5XaB95Yv3ACcLekvwNvAj2o/VTMzM6u2XI9wRsRbpBt+difduf1r4DrS3dcA1wKPAS+Qpom3poX1fW30IOmGpSmS5kraOSKmAHsCQ4A3SDezDIuIm2us8ybSqF534M7C9sp5PU4aofs20GwwjIgXgeOBK4A3gf8GbiuULwR2JV2P54C3gN+Qbvppjxavc0S8DQwDbszX6vQm+tzea9cNOBaYKmkecClwWERMzeXnAodImiNpYt52PGka/Tngz6TR4tbWrZqZmVkz6hobG1vfy8xqUnfBog/9g2o8aXmfSDAzsxVYTcsQl+sRTjMzMzPrfB56aSdJOwITmyk+LyLOa2N9pwGnNVO8R0R01FT/R9LR590RlpVrZ2ZmtqLylLpZB2poaGisr6/v7G6YmZktLZ5SNzMzM7PO58BpZmZmZqVy4DQzMzOzUjlwmpmZmVmpHDjNzMzMrFQOnGZmZmZWKgdOMzMzMyuVA6eZmZmZlcqB08zMzMxK5cBpZmZmZqVy4DQzMzOzUjlwmpmZmVmpHDjNzMzMrFQOnGZmZmZWqpU7uwO27JI0CXggIs7t7L5USPohcBCwLvAv4HfAiRExvbDPZsAFwK5507PAjhHxbi4XcBnwGWAmcFZETFhqJ2FmZrac8QindRpJq5RQ7Xjg8xHRE+gLTAduKrTZC5gMPAVsAqwDDAEW5/I1gYnAbcDawNHA5ZJ2KKGvZmZmKwSPcFq7SLoE2BHYQdKpwCsRsaWkI4Hjgd7AS8DQiLgvHzMc2Al4EjgUeFLSMcAUYBAwFOgDPAQcnN8fASwBzomIS1vrV0Q8V3hbl4/dsrDtRGB6RAwvHlZ4vS+wEDg/IhqB+yX9Evgf4JHW2jczM7MP8wintUtEDCGNFJ4TEd0LYXMoKSyuDZwO3C5p88KhO5GmqXsD+xW27wd8jTTq2Bd4DHgR+CRwOHCRpE1q6ZukgyS9Bcwnhd/hheKvAzMk3S3pTUlPSzq4UL4N8MccNiuezNvNzMysHTzCaR3peGBERDyV398j6UHgAKCyznNaRFyYX7+TlksCKbi+CSDpLmCviLgyl02UNAfYljRF3qKIuAG4QdKGwHeAPxeK1wO+BAwA9iEF0AZJ0yLiYaAH8FZVlXOBnq2evZmZmTXJgdM6Uj/gUkkXF7atDLxceD+tmWNnFl4vrHpf2dajLZ2JiFmSrgRekrRJDrTzgEci4ta82/2S7gW+ATycy/tWVbUW8M+2tG1mZmbvc+C0j2JJ1ftppDu6b2nDMWVbGViDNDX/JvAnYPMm9qtMoT8FfLOqbNu83czMzNrBgdM+ill8MLyNAYZL+jspoK0GfBF4vepmnlJI6gZ8F7g5Il6TtDHwU2AqUGl/LDBZ0jeBXwE7A/2BUbn8l8D5kk4GfkK6MWpf4D/L7r+ZmdnyyjcN2UcxhvTYyrmSnslrLs8HrgHmkNZbDgPKePxRc/YE/iJpAenGo4XA7hGxCCAiHiU9p3MUafr8p8BhEfFILp+b69iftJbzSuDoSrmZmZm1XV1jY2Pre5lZTRoaGhrr6+s7uxtmZmZLS10tO3mE08zMzMxK5TWctkyRdDlwSDPFWxf/hKWZmZl1DQ6ctkyJiKNJf27SzMzMlhGeUjczMzOzUjlwmpmZmVmpHDjNzMzMrFQOnGZmZmZWKgdOMzMzMyuVA6eZmZmZlcqB08zMzMxK5cBpZmZmZqVy4DQzMzOzUjlwmpmZmVmpHDjNzMzMrFQOnGZmZmZWqnYFTkkbS2qU1LeGfYdLeqA97SwNki6XdEln96Mr68xr1BFtSzpY0lMd1SczMzNrm5U7uwM5tE4BekfEyzXsPwg4IyI274j2I+LojqhneSFpKun6Tqhs68xr1BFtR8T1wPUd0B0zMzNrB0+pryAkrdLZfTAzM7MVU00jnJI2BK4AdgZeBc6vKj8SOB7oDbwEDI2I+wq71EkaAwwE3gYuiYgf5bLKVOffJDUCoyLinGb6sQNwObCqpPl5894RMUnSzrlfWwEzgTERMbaGcxsHLIqIwfl9I3AscHiu6xlgUEQ8l8tXAU4GDgM+CbyWz/dWSSsDpwGDgLWBJ4HjI+IvhbZWAt4F9gUWACcBzwJX5vYCODgi/pGPWR0YAewHrAk8DgyJiBdaOa9dgAfyeZwN9AJ6SDoeOAbYCJhDGvk7IyIWS2oANgGuknQ58PuI6N/ENeoDXAx8lfR53gb8ICLebqVPw4Ed8zkeQfqF54f5+GuALwHPA4dExLOFa7YoIgZLqgPOzefUA3gDuDAifippbdLP6K6kn+uXgaMjYnL1qLikScATQF+gP+kzPDEi7szldcAPgO8CqwPXAp8DJkfE8JbO0czMzD6s1hHO64HFpDCyEylQAe+FzaHAwaSQdTpwu6TilPdOpKD6CWAf4ERJB+WybfL3LSOie3NhEyAiHgGOBl7K+3bPYbMfcC/wM2Dd3L+Rkvav8fyqDSIFvPWAGcBPC2XnAocA+wM9SSH8+Vx2MilU7wlsCEwG7pfUs3D8t0kBax3gHFLQHAF8C9gAaCQFxIpKEN0+1/kYcFeNI5Yr5b5sm+uGFMT2yH3fhxT8BgNERD0wHRicr23/6gpzqL4bmAX0yf36KnBBDf2B9LPw93wuhwCjgatJIX8dUvi+uJlj/5MU9LeLiB7Al4GHc9nJpHDYB1iLdD1bWqJxGHAhKcRfAlybwz3AoaRfoOpJ121m7reZmZm1Q6sjnJI2Io0abR4RbwFvSTobqIxgHg+MiIjKSOU9kh4EDiCFM0j/wx4VEY3AE5KuIIW6GzroPA4EnoyIcfn9o5LGkoLULe2ob3RETIf3Rtgm5Nd1pGA0ICKezvu+zPvB5nDSeVZGQ0fkPuwF3Jj3+W1E3J3LryOF5PGV9auSbgWOzK/XAw4C+kTEq3nb2cAJwHa8H7ZaMjR/bgBExG2Fsj9KGg/sBrQ6Gpx9GdiCFPoWAAsknQHcIWlI/oxb8nxEXJVfT5T0BvDrwojmDTS/3vIdYDXg05JmR8RrpNHJStm6wJbAHyPi+WbqqPhFRPw+t3kF8ON8Xk+RfmkYGxF/zOWjSZ+7mZmZtUMtI5wb5+/TCtumFF73Ay6VNLfyBXydNGVbMa0qiEwt1NsRelf1CeDFvL09ZhZeLyBN30Kall6D90c0W+xHRCwhnWuxHzML5QubaG9hob1++fvThWv7JrAKtZ3bEtII7XskHSjpD5LekPQWKUj1qqGuit7A7Bw2K14kBcFa6plZ9X4hzZ//B0TEJNKShTOA1yTdJ0m5eDTwG9L092xJ10raoKl6qvtROJdKuxtR+HnPP7sfuI5mZmZWu1rWcL6Sv/chBQtIa98qpgFnRURLI4l9JNUVQmdf3h8VXFJbV9/T1P4zSFPHRZvS8SFhNikQbUGaFm6qH30rbyR1y+/b249K6NkiIma34/jGYtCX1Js0WrsvMDEi3pF0AaDCMa19HjOAXpJWLwTmTYF/ka5PqSLiCuCKPP09HLgd2CSHxtOB0/Oa4wmkEDqwHc28Qvp5B94b2W7vLy9mZmYrvFYDZ0S8nG+yOF/S4cDHgTMLu4wBhkv6O2k6cjXgi8Drlall0trNk/ONQ58hTRmfmMtmk0LOFrS85q5iFrC+pJ4R8c+87UZgmKSBpGn6LwBHkW6O6TAR0SjpMtK1mE66oWgjYJ08xT4OOEXS70gjm0NJ1/judrb3Wp5ivkzSCRHxiqS1SCPI90fE/FaqqNadNKo9G3hX0vak9YrPFvaZRfosmvM48AJwoaT/Ja2XPAe4pobp9I9E0peBj+U+/BuYR1pbjKT63K/ngfmkALy4nU2NB0ZJup30GR9HukHMzMzM2qHWm4YOIv2PfgbpRpjrKgURcSXp7vBrSHc9TweGkaZ9KyaTQucs4C7gJ+T1m/nO5mHAjXna+PRW+vIgcD8wJe+/c0RMIY1wDiHduTweGBYRN9d4fm1xOnAzcAcp8EwCKjdIjSaF3/tIN0ntCvQvBOP2OBL4GzBJ0jzgz6Qbltoc7vI6ybOAO4G5wKm8v7a04lzgEElzJE1soo5FwN6kJRHTSeHvMdLd9mXrTvrZeZ30OfcHBuSyzYAG4J+ksP82KfC3x3XApcA9pM9xY+BRUsg1MzOzNqprbCx1UMpsmZeXRswATo6IFm90a2hoaKyvr186HTMzM+t8dbXs1Ol/acisK5J0AGkUuxvpmZyrAx8a8TUzM7PWdbnAKWlHmv8f+3kRcV4b6zuNdGdzU/aIiMltqa8rKTz8vtrkiNhjqXaGjv/sOtkQ3n9U1F+APSNiTif2x8zMbJnlKXWzDuQpdTMzW8HUNKXuv6VuZmZmZqVy4DQzMzOzUjlwmpmZmVmpHDjNzMzMrFQOnGZmZmZWKgdOMzMzMyuVA6eZmZmZlcqB08zMzMxK5cBpZmZmZqVy4DQzMzOzUjlwmpmZmVmpHDjNzMzMrFQOnGZmZmZWqpU7uwO24pB0KvB9YA3g6xHxh6XU7uXAoogYsjTaMzMzsw9y4LSlQtLGwHnAZyLir0uz7Yg4emm2Z2ZmZh/kKXVbWvoCSz5K2JS0Ssd1x8zMzJYWj3DaB0jqDgwH9gV6ATOAo4AngJF5+8eBh4HjImJ6Pm5S3qcv0B94DTgxIu6UNAAYB6wkaT7wakRsJmldYEzeH+DXwPcj4s1c51Tg58DXgS8BgyVtBewIBHAE6ZemHwK3Adfk/Z4HDomIZ3M940hT6oPz+0bgWOBwYCvgGWBQRDyXy3sAlwJ7A/OAYbkfu0fEpI9yfc3MzFZEHuG0alcD2wG7AT2BbwAzScFw+/zVB3gdaJC0UuHYw4ALgTWBS4BrJa0eEb8A9gAWR0T3iNgs7389sDbwH/lrPWB8VX+OBE4EegB35m07AX8HNgQOAUbnfh8L/7+9Ow+XqyjzOP59ycIWAiiJAiEkkYA6EJUUPDgCxoVNRFFBBALGKCaOG/DI6ETQKCiLICMzMKwmLMoiM8IEhIAIgzIjWAIKskRCbgIhMQGSmBC2hJo/qjqpe9Ldt/uml6Tv7/M8/XSfrU6dOnXOebtOnW7eBDwOXNDDdk4APpXW+Qzwb9m0nwCjiMHoHsChQD9ERESkV9TCKWs454YCnyb2s5yTRj/lnNuEGEwe5r2fn+Y9EXgR2Bv4vzTv9d77/03TLwV+DIwG/lRmXTsABwG7eu+XpHEnA08457b33i9Is17mvX8ofX7ZOQcwy3t/eRp3m3PuBWBm1qL5c2IwW82PstbZ6cA16XM/4FjgEO/9ojRuSioXERER6QW1cEpuRHqfVRg/BNgUKAWheO9XEG+b75TNtyCb/lL6uFWFdZWWm5ONm12YBtBVZtkFheGVhXErq6y3XBovZfNvBwwE5mbT888iIiJSJwWckutK76ML4xcDr7I2IC319RxKvB3dG6XlRmTjRhWmAbzRy/R763ngNWK3gZLhLc6DiIhIR9EtdVnDe7/IOXcjcJFzbgKxZa/U3/Iq4HTn3GPAUmJfzSeAB3q5ruecc3cA5znnPgtYSvO27HZ6y3nvV6db8lOdc48ArxAfShIREZFeUgunFE0EHgb+h/iE9s3Eh3NOIj4Z/gdgHrA98DHv/er1WNf4tI4nicHrUuD49UivUb5O3MZZwKPAnUAgtvKKiIhInSyE0O48iGzQnHO7EQPiHb33z1Wbd8aMGeGwww5rTcZERETaz2qZSbfURQqcc6OIrbr3Ex8iOh+4t6dgU0RERMrTLXWRdW0GXAosAx4hPvV+TFtzJCIishFTC6dIQfr7zd3bnQ8REZFOoRZOEREREWkqBZwiIiIi0lQKOEVERESkqRRwioiIiEhTKeAUERERkaZSwCkiIiIiTaWAU0RERESaSgGniIiIiDSVAk4RERERaSoFnCIiIiLSVAo4RURERKSpFHCKiIiISFMp4BQRERGRplLAKSIiIiJNpYBTRERERJpKAaeIiIiINJUCThERERFpKgWcIiIiItJUFkJodx5EOsamm2766GuvvfZKu/PRF/Xv33+7VatWPd/ufPRFKvv2Udm3j8p+jedDCAf3NFP/VuREpK/YY489XvHeu3bnoy9yznmVfXuo7NtHZd8+Kvv66Ja6iIiIiDSVAk4RERERaSoFnCKNdWm7M9CHqezbR2XfPir79lHZ10EPDYmIiIhIU6mFU0RERESaSgGniIiIiDSVfhZJpE7OuV2BK4E3Ay8Ax3vv/1qYpx9wAXAwEICzvPeXtzqvnabGsj8N+AywGngdmOK9n9nqvHaaWso+m3c34CHgIu/9N1qXy85Ua9k75z4NnAYY8bzzYe/931qZ105T4zlnKDAN2AkYANwNfM17v6rF2d2gqYVTpH4XAxd673cFLgQuKTPPscAuwGjgvcBU59yIluWwc9VS9g8Ae3nvxwATgeudc5u3MI+dqpayL33ZugS4qYV563Q9lr1zzgFTgQO897sD+wLLWpnJDlVLvZ8CPJ7OOWOAscAnW5fFjYMCTpE6pG+yewLXplHXAns654YUZj0KuMx7/4b3fjHx4ntk63LaeWote+/9TO/9yjT4Z2Jrz5tbltEOVEe9B/gWcAswq0XZ62h1lP1JwLne+4UA3vtl3nv969l6qKPsA7CVc24TYFNgIDC/ZRndSCjgFKnPTsB87/1qgPT+XBqfGw7MzYbnlZlH6lNr2eeOB2Z7759tQf46WU1l75x7F3AQcH7Lc9i5aq337wRGOefudc496Jw71TlnLc5rp6m17E8HdgUWAAuBmd77+1qZ0Y2BAk4R6UjOufcTLwRHtzsvfYFzbgDxdwknly7Q0lL9iLdzDwDeDxwCHNfWHPUdRxLvpmwP7Ajs75w7or1Z2vAo4BSpzzPAjqmfWqm/2g5pfG4esHM2PLzMPFKfWsse59x7gWuAw733T7Y0l52plrLfHngb8CvnXBdwInCCc04/jr1+6jnn3Oi9f9V7vxy4Gdi7pTntPLWW/VeBn6UuVMuIZf+BluZ0I6CAU6QO3vtFwMOsbTU7Gngo9dPM/YJ4sd0k9fc5HLixdTntPLWWvXNuL+B64Ajv/YOtzWVnqqXsvffzvPfbee9HeO9HAP9K7Mf8xZZnuIPUcc75OXCgc85Sa/OHgD+1Lqedp46yn0P8RRKccwOBDwOPtiqfGwsFnCL1mwx81Tk3i/jNdjKAc+5X6UlRgKuBp4G/Ar8Hvu+9n9OOzHaYWsr+ImBz4BLn3MPptUd7sttRail7aY5ayv46YBHwGDFI+gtwRRvy2mlqKfsTgf2cc48Qy34WcFk7Mrsh019bioiIiEhTqYVTRERERJpKAaeIiIiINJUCThERERFpKgWcIiIiItJUCjhFREREpKkUcIr0YWZ2kJn9NhseZ2ZdbcxSy5jZdDO7vIHpjTCzkA0PMbO5ZrZdDctONrOrG5WXjYGZ7WdmS9udj77IzMbXc5w3+liR6pp1bPRiv59lZqc3av0KOEX6KDMz4n9ef7eH+b5kZo+a2d/NbImZeTM7KpveZWbjyyy3zniLZqW0BhWmjTOzYGYr0us5M5tmZm9avy1tjxDCYuKPcfdUvlsC3wemtiBbG4wQwm9DCNu0Ox+VmNlUM/t1u/PRFzSrrM3sHjM7tdHpNlvx2GhjXTwb+LKZ7diIxBRwivRdBwIDgbsrzWBmRxMDps8DWxP/1u0kYEkv1/kBYBTwBuX/43x1CGFQCGEQsC/wXuI/1mysfgp8zswGV5lnPPBICGF2i/LUjZn1MzNdC0SkmxDCEuA2YFIj0tNJRqQFUmvfqWZ2d2q9e8TMxpjZ0Wb2lJktM7PLzax/tsxwM7vRzBaa2QIzu9TMtsqm/9DMnk7pzTazE7NpI1Jr4XFm9piZLTezO8xs+yxbhwO/DtX//eEfgXtDCPeH6OX07fuOXhbFJOB24j8xVT2JhRCeBm4B3lOcZmb9U5kcXhg/3cympc8fMrP7U6vsYjO7zsyGVlpfKq99s+FxZraqsM4pqYV2qZndZ2ZV/2EnhPBX4HniX91VcjhwZyEvXzezJ9J+m2dmZ5pZvzTtR2Z2U2H+cWneLdPw7mY2M213afkBaVqpbnzezB4DVgJDzewzZvan1Pq8wMwuKaWXlnurmc1IdXVWWj6Y2YhsnhNSa/gyM3vIzGt6vHsAAAuUSURBVA6stNFlyne6mV1tZj9N5Ts/HR/vNrM/pO2728x2yJbpMrPvmNnv0nHgzWyvbHrVOmBmA9I+fTKlP9vMjrDYgj8FGGdrW9xHVdiO96d1LEv7bFI2bZyZrTKzo1Lay8zshvw4LpNeb84VY8zsN2k7n07L98um753KZoWZ/Y74pS9f5xZmdq6ZzTGzF83sdjPbpVIey+T5zWZ2lcVz1UIzu9KyOxNWuNuR1cFhlcrazCak7f1mqo+LzOy8MvV4WJbuBDN7Kn3+d2A/4LSU5pMV8j7VzO4ys7NTHXnBzE42s51TmS43sz+a2TuyZdbrWMnq+mVZXV+n3qTPVcunsC3duj40aL/fSTxHrb8Qgl566dXkF9BF/JvLdwADgGuA2cClwJbAcOLf0h2b5t8MeIp4q3VzYFvgV8BPszTHE1scDfgg8DJwUJo2AgjEgG07YDBwH3BZtvz9wNcK+RwHdGXDRwKvAGcQ/5t5mwrbNr6n8cAQ4FXgk8QgMgBjC+telQ3vAjyZb3Mh/XOAm7LhQcAKYL80vC+wF9AfeCtwL3BtNv904PJsOAD7VsnPD1KZjQL6EVt9nwe2zcu8TD5nAGdUqRt/Az5WGPcpYGTat+9J80xK094JvAYMyea/ErgifR4KvEAM6AcCOwIe+E6hbtyVymVg2p5DgH8gNkTsQvyLxDOzddwF/GeqS0OBe1I6I9L0E4h19l0pjY+k/bFLhe0ulu90Yh0+NC0/OS3/38AwYAvgN3Svw13Ac8DYtB3fAhYDg2usA2en7RyTynoYMCZNm0r8QlbtuB6Z8jwhrWMf4EXgyGwbA/EvJgcBbyGeB77dwHPF1ql+nAZsmpZ7Gjglm/5CKpuBqTwW0v04/xnxXPGWNM/3gCeAAeWOlTJ5vp1Yz7dNr1uBW6ucC0akchlWqaxTmb4OXEg8B76N+JeRU8qlkS3zVDZ8D3BqD/twalrPF1h7HKwGfl3YB3dmy6zvsTKdWG8+ltL4ZMrDzhWOjUrl81Rh3Jr91Ij9nuYZS7wjNbBaOdbyaulFVy+9+uornXBPyYY/kk5AedBwA3B++nwEMLuQxlhiwNavwjpuBM5Jn0sn472y6V8GHsqGZwETCmmMy09IadxHgf8iXtRWE2/B717YtpeApYXXG3S/yPwz8UJZuog9CFxSWHdIyy4B5gAXUybITfO/gxh4DU3DE4FZVfbBR4FF2fCak3MarhhwEoOR5cD+hTQfKW0jlQPOnwEXVcnXa8C4HurPucAN2fD9wEnp81ap/N+Xhr8B/Kaw/KdIF6esbuzfwzq/AjyQPg9Ly4zKpn+I7hfRR4HjC2nMoMIFn/IBZx6kbJHSPzIb9090r8NdwOnZsAHzgGN6qgNp3hXAoRXmnUrPAecU4L7CuDOBmYU6nR/nPwJ+WSXNLuo7VxwDPEP6q+o0bhLwZPp8bCqTfPoPSMc58QtpAIZn0zcBlpGOB6oEnMQvvQEYnY3bLY3bPtum3gScrwJbZOO+QDrGi2lky/Qm4PxLYdyiMvtgSQOPlelkdT2NWwx8vMKxUal8qgWc673f07jRab6h1cqxlteaJnkRaboF2eeVxP6KiwvjSrfaRgLDbd0nFQOxpWa+mX2N2Ko0jHjx3Jz4kEqldb6UpQ8xqKvWtzCuMIRbiN+CMbO3AxcBt5jZyJDOSMTWt2vy5Sx7GtLMLOX1mhDC62n0FcBZZvaNEMLyNG51qPFBkhDC42b2ILGl98fA54Bp2TrHAj8ktrhtQSyjQWWSqsV2adkZlj2JTmz9GFZ+kTUGE4PnStbZDxb7zp5MbE3tT2x9+H02yzTgS8SHvj4NPBtCuC9NGwm8r1B3jNh6k+sqrPMA4DvA24ktZf2IF16IraQQL2AlcwvpjQQuNLMLsnH9gWep3Zr6GkJYGavNOsdN8XZ0V7ZMMLN5pH3SQx0YQmwxnFVH/op2Yt19Oxv4eDZcPM6Lx2E59ZwrdgLmZsdiKQ87pc/DykzP8zwyvf85lXfJgCyNakrz5GnOzqYtoPcWhRBWZsNd9Hy89UYxjyupUu8acKyUW2ct9aIejdrvg1nbELBe1IdTZMM0l/hNfpvCa7MQwnwzex/xduAkYLsUpM0gXlBr9RDx9mzNQghPEIOcnYm3zmr1QeKtp4mlfl7E2zeDiC00vTUNmJD6He0DXJVNu47YirprCGEw5R9Syq0gBiAlO2SfnydeED5c2B9bhhDO6iHd3YllXUm3/WBmOxFv4Z1BbCHamnhbMd+31wG7mtmexJaOadm0ucTWkDyfW4f4IFbujWydA4GbUrrDU3l9M1vn/PQ+PFs+/1xa78TCegeFEL5UZdsbYUTpQ/piM5y1QW61OrCYGEiMrpDuGxXG557J15+MSuNb5RlgZ+seNeR5mF9m+ojscykYGl3Yd1uEEK6tcf3FNEcVpi2n8rEFlct6qJltUch3ad+WvqT2Jt1ea9CxUq9y21EsU+i+/Y3a77sTW4Bf62Xe11DAKbJhugUYaPGBhq0s2tHMPpGmDybe3l4MBDM7lNivqB43EW/1VGRmE83sSEu/JZk66E8GHgshvFjHuiYR+8+9HXh3eu1ODJS+WGe+c9cRA9kLiH2s5mfTBhNvDy03s+HEvkzV/BH4rJkNTJ37Ty5NSK0EPwHONbPRAGY2yOLvmBYvcmukQHgIsT9YJTfR/aGiQcRz82LgdTPbBzguXyCEsBT4JTEo3YfYh7PkKsClfbeZmW2SHjI4uEoeBhJbapaEEF42s3cSbxOW1vcs8fbkWak+DgGKPzdzPjDV4kM+Zmabm9m+qVW8mSaa2Z4WHyY5hdiSeWuaVrEOpH16EXCOxYeszOJDLGPSLAuJdxkGVln3tcBYMzve4kNlexPr+hUN3cLqbiXuuymp7u5GDIBKebiFWKdOsfiQ1J7E/scAhBAWEe+MXGTp52/MbBsz+4QVfrqsnBDCc8AdwHlpuW2B84DbQgilVrw/AkenY2YIsb9prlJZbwKcnerSKGJ3kSvTel8gfcmx+EsLexDvohTTrfnhpxo14lipV7nyeZgYkH80HeOfAPbPpjdqvx9APEetNwWcIhugdBvpg8SWryeIF827iIEawExiYPEAsfXtCGIAUo+ZwCozG1dlniXEW7ePm9lLxL6DS4l94Wpi8angw4FzQwgL8xexlfY91sPT3pWEEJYRt/sQ4k8Q5b5I7PO1nNgH9Rc9JPcV4sXpRWIfuemF6d8FbgZuNrO/Ex/smEz18+hEYHrKZyVXA+9KF1RCCI9n61pKDJLKtTRNI273zOzCTirXDxDLvIu4D39J4QnVXAhhBXE/n2NmK4gtqsXuGccQg7lniQ+glcrz1ZTGZcQHuaaldc4jBhYDqmx7I1xK/MKxBDiK2CezVN491YFvE/f1TWmee1gboPyC2EK30OKTxCMLyxJCmEPs3/cV4gMaVwOnhRBuaNTG9SRt64HELy1/Y+254cdp+lLig1hHEcvoAuA/CsmcQHxA7x4zW07sm3wk8VZqLcYTy+9J4vlqKXB8Nv1U4hfkBcQyvq6wfKWynkusb3OI557biXWs5LPEc9GytL3FQP984pevpWb2lxq3papGHCu9sE75hPgzal8n1v8XgYOJDyqV8rne+93MtiHW74t7me9urPvtfRHpS1Kr15QQwv5peBwxQBrRznxtjFKr6JwQgqXhIcSnw12h/125ZScTH/o5rtp8GxIzO4gYFG8e2nQhsdhP+NRi/2HZ+JnZBOK+bXQLZcttCMdKb5jZmcT+ww358Xw9NCTSh4UQbie2GkiDpSBz5xrnvZgGtSI0i5m9m9iX7BHiAwdnANdvTBdQkVbolGMlhPAvjUxPt9RFJNfFxv3PPu20lPggVKfalnhbegXwO+DPxFt6ItKdjpUydEtdRERERJpKLZwiIiIi0lQKOEVERESkqRRwioiIiEhTKeAUERERkaZSwCkiIiIiTfX/MaDyGeDhek4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 576x396 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "shap.summary_plot(shap_values_xgb_default, X, plot_type=\"bar\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MGBM Grid Search (Constrained)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "# GBM hyperparameters\n",
    "hyper_params = {'learn_rate': [i * 0.01 for i in range(1, 11)],\n",
    "                'max_depth': list(range(2, 11)),\n",
    "                'sample_rate': [i * 0.1 for i in range(5, 11)],\n",
    "                'col_sample_rate': [i * 0.1 for i in range(1, 11)]}\n",
    "\n",
    "# Search criteria\n",
    "search_criteria = {'strategy': 'RandomDiscrete', 'max_models': 36, 'seed': 1}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gbm Grid Build progress: |████████████████████████████████████████████████| 100%\n"
     ]
    }
   ],
   "source": [
    "# Train and validate a random grid of GBMs\n",
    "mgbm_grid2 = H2OGridSearch(model=H2OGradientBoostingEstimator(distribution='bernoulli',\n",
    "                    # monotonicity constraints\n",
    "                    monotone_constraints=mono_constraints,\n",
    "                    # more trees is better if the learning rate is small enough \n",
    "                    # here, use \"more than enough\" trees - we have early stopping\n",
    "                    ntrees=10000,\n",
    "                    # learning rate annealing: learning_rate shrinks by 1% after every tree \n",
    "                    # (use 1.00 to disable, but then lower the learning_rate)\n",
    "                    learn_rate_annealing = 0.99,\n",
    "                    # score every 10 trees to make early stopping reproducible \n",
    "                    # (it depends on the scoring interval)\n",
    "                    score_tree_interval = 10,\n",
    "                    # fix a random number generator seed for reproducibility\n",
    "                    seed = 1234,\n",
    "                    # early stopping once the validation AUC doesn't improve by at least 0.01% for \n",
    "                    # 5 consecutive scoring events\n",
    "                    stopping_rounds = 5,\n",
    "                    stopping_metric = \"AUC\",\n",
    "                    stopping_tolerance = 1e-4),\n",
    "                    grid_id='mgbm_grid2',\n",
    "                    hyper_params=hyper_params,\n",
    "                    search_criteria=search_criteria)\n",
    "mgbm_grid2.train(x=X, y=Y,\n",
    "                training_frame=train,\n",
    "                fold_column=fold_column,\n",
    "                ntrees=100,\n",
    "                seed=SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get the grid results, sorted by validation AUC\n",
    "mgbm_gridperf2 = mgbm_grid2.get_grid(sort_by='auc', decreasing=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "          col_sample_rate learn_rate max_depth         sample_rate  \\\n",
      "0                     1.0       0.09        10                 0.9   \n",
      "1                     0.8       0.09        10  0.7000000000000001   \n",
      "2      0.7000000000000001       0.09        10                 0.5   \n",
      "3      0.7000000000000001       0.08         9  0.6000000000000001   \n",
      "4                     0.5        0.1         8  0.7000000000000001   \n",
      "5                     0.9       0.08         6                 0.8   \n",
      "6                     1.0       0.09         5                 0.5   \n",
      "7                     0.9       0.06         6                 0.9   \n",
      "8                     0.8       0.05        10  0.6000000000000001   \n",
      "9     0.30000000000000004        0.1         6  0.7000000000000001   \n",
      "10                    0.4       0.08         5                 0.8   \n",
      "11    0.30000000000000004       0.08         6                 0.8   \n",
      "12                    0.4       0.07         5                 0.5   \n",
      "13     0.7000000000000001        0.1         3  0.7000000000000001   \n",
      "14                    0.9       0.04         7                 1.0   \n",
      "15                    0.2       0.09         6                 1.0   \n",
      "16     0.7000000000000001       0.04         8                 1.0   \n",
      "17    0.30000000000000004       0.04        10                 0.9   \n",
      "18     0.6000000000000001       0.05         4                 0.9   \n",
      "19                    0.2       0.06         7                 0.8   \n",
      "20                    0.9       0.06         3                 0.8   \n",
      "21     0.6000000000000001       0.04         7                 0.8   \n",
      "22                    0.9       0.04         4                 0.5   \n",
      "23                    0.4       0.04         9                 0.5   \n",
      "24                    0.2        0.1         3                 0.8   \n",
      "25                    0.2       0.06         4                 0.9   \n",
      "26     0.7000000000000001       0.08         2                 0.5   \n",
      "27                    1.0       0.07         2  0.6000000000000001   \n",
      "28                    0.1        0.1        10                 0.5   \n",
      "29                    0.2       0.03         7                 1.0   \n",
      "30    0.30000000000000004       0.06         2  0.7000000000000001   \n",
      "31                    0.1       0.09         3                 0.9   \n",
      "32                    0.4       0.01         6  0.6000000000000001   \n",
      "33                    0.1       0.02         6                 0.5   \n",
      "34                    0.1       0.01         6  0.6000000000000001   \n",
      "35                    0.1       0.01        10                 0.5   \n",
      "\n",
      "              model_ids                 auc  \n",
      "0   mgbm_grid2_model_28  0.8089712551390209  \n",
      "1   mgbm_grid2_model_26  0.8084536803079319  \n",
      "2   mgbm_grid2_model_36   0.807873467978609  \n",
      "3   mgbm_grid2_model_27  0.8071308310379903  \n",
      "4   mgbm_grid2_model_20  0.8064779893848008  \n",
      "5   mgbm_grid2_model_11  0.8057817301810621  \n",
      "6   mgbm_grid2_model_24  0.8050722593506725  \n",
      "7    mgbm_grid2_model_7  0.8042427196221075  \n",
      "8    mgbm_grid2_model_4  0.8042158506152867  \n",
      "9   mgbm_grid2_model_12  0.8027726338680145  \n",
      "10   mgbm_grid2_model_3  0.8022820211131623  \n",
      "11  mgbm_grid2_model_16  0.8018629308686189  \n",
      "12   mgbm_grid2_model_6  0.8009621200062299  \n",
      "13  mgbm_grid2_model_18   0.800891905143482  \n",
      "14  mgbm_grid2_model_35  0.7994876321578194  \n",
      "15  mgbm_grid2_model_22  0.7994464302382729  \n",
      "16  mgbm_grid2_model_34  0.7990698598029258  \n",
      "17   mgbm_grid2_model_2  0.7979616800394912  \n",
      "18  mgbm_grid2_model_30  0.7977507010247809  \n",
      "19  mgbm_grid2_model_31  0.7975923785927225  \n",
      "20  mgbm_grid2_model_17  0.7975011601002516  \n",
      "21  mgbm_grid2_model_33  0.7974681678828788  \n",
      "22  mgbm_grid2_model_13   0.796973476104816  \n",
      "23  mgbm_grid2_model_19  0.7968711131064086  \n",
      "24  mgbm_grid2_model_25  0.7962833790760978  \n",
      "25  mgbm_grid2_model_14  0.7956792015947965  \n",
      "26  mgbm_grid2_model_15  0.7944709425961134  \n",
      "27   mgbm_grid2_model_5  0.7938489605792753  \n",
      "28   mgbm_grid2_model_9  0.7920160686439757  \n",
      "29  mgbm_grid2_model_29  0.7917487338713067  \n",
      "30   mgbm_grid2_model_8   0.789154197237066  \n",
      "31  mgbm_grid2_model_21  0.7885843585851671  \n",
      "32   mgbm_grid2_model_1  0.7850942345538496  \n",
      "33  mgbm_grid2_model_32  0.7766357487023613  \n",
      "34  mgbm_grid2_model_23   0.772165363698924  \n",
      "35  mgbm_grid2_model_10   0.771420520256121  \n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(mgbm_gridperf2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8139763217062082"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Grab the top GBM model, chosen by validation AUC\n",
    "best_mgbm2 = mgbm_gridperf2.models[0]\n",
    "\n",
    "# Now let's evaluate the model performance on a test set\n",
    "# so we get an honest estimate of top model performance\n",
    "best_mgbm_perf2 = best_mgbm2.model_performance(test)\n",
    "\n",
    "best_mgbm_perf2.auc()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Shapley Importance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "First 5 rows of contributions:\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead>\n",
       "<tr><th style=\"text-align: right;\">    term_360</th><th style=\"text-align: right;\">  conforming</th><th style=\"text-align: right;\">  debt_to_income_ratio_missing</th><th style=\"text-align: right;\">  loan_amount_std</th><th style=\"text-align: right;\">  loan_to_value_ratio_std</th><th style=\"text-align: right;\">  no_intro_rate_period_std</th><th style=\"text-align: right;\">  intro_rate_period_std</th><th style=\"text-align: right;\">  property_value_std</th><th style=\"text-align: right;\">  income_std</th><th style=\"text-align: right;\">  debt_to_income_ratio_std</th><th style=\"text-align: right;\">  BiasTerm</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td style=\"text-align: right;\">-0.00018562 </td><td style=\"text-align: right;\">  0.00525494</td><td style=\"text-align: right;\">                   -0.00119815</td><td style=\"text-align: right;\">        0.107912 </td><td style=\"text-align: right;\">                 0.887901</td><td style=\"text-align: right;\">               -0.00580012</td><td style=\"text-align: right;\">            -0.00468927</td><td style=\"text-align: right;\">           0.527515 </td><td style=\"text-align: right;\"> 0.0166724  </td><td style=\"text-align: right;\">                  0.272783</td><td style=\"text-align: right;\">  -2.91142</td></tr>\n",
       "<tr><td style=\"text-align: right;\">-0.000157597</td><td style=\"text-align: right;\">  0.00815592</td><td style=\"text-align: right;\">                   -0.0021666 </td><td style=\"text-align: right;\">       -0.146934 </td><td style=\"text-align: right;\">                 0.536647</td><td style=\"text-align: right;\">               -0.0180488 </td><td style=\"text-align: right;\">            -0.00987545</td><td style=\"text-align: right;\">          -0.0619093</td><td style=\"text-align: right;\">-0.00628095 </td><td style=\"text-align: right;\">                 -0.29549 </td><td style=\"text-align: right;\">  -2.91142</td></tr>\n",
       "<tr><td style=\"text-align: right;\">-0.00274492 </td><td style=\"text-align: right;\">  0.00705713</td><td style=\"text-align: right;\">                   -0.00344599</td><td style=\"text-align: right;\">        0.388739 </td><td style=\"text-align: right;\">                 0.620492</td><td style=\"text-align: right;\">               -0.14514   </td><td style=\"text-align: right;\">            -0.943326  </td><td style=\"text-align: right;\">           0.0448351</td><td style=\"text-align: right;\">-0.0277831  </td><td style=\"text-align: right;\">                  0.018798</td><td style=\"text-align: right;\">  -2.91142</td></tr>\n",
       "<tr><td style=\"text-align: right;\">-0.000380784</td><td style=\"text-align: right;\">  0.012291  </td><td style=\"text-align: right;\">                   -0.00452263</td><td style=\"text-align: right;\">       -0.0809743</td><td style=\"text-align: right;\">                -1.31932 </td><td style=\"text-align: right;\">               -0.0377705 </td><td style=\"text-align: right;\">            -0.0840968 </td><td style=\"text-align: right;\">          -0.308984 </td><td style=\"text-align: right;\">-0.000403701</td><td style=\"text-align: right;\">                 -0.190623</td><td style=\"text-align: right;\">  -2.91142</td></tr>\n",
       "<tr><td style=\"text-align: right;\">-0.000354039</td><td style=\"text-align: right;\">  0.00488184</td><td style=\"text-align: right;\">                   -0.00119556</td><td style=\"text-align: right;\">        0.354748 </td><td style=\"text-align: right;\">                 1.07438 </td><td style=\"text-align: right;\">               -0.00380643</td><td style=\"text-align: right;\">            -0.00236983</td><td style=\"text-align: right;\">           0.153142 </td><td style=\"text-align: right;\"> 0.0352928  </td><td style=\"text-align: right;\">                  0.256853</td><td style=\"text-align: right;\">  -2.91142</td></tr>\n",
       "</tbody>\n",
       "</table>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "shap_values_mgbm, expected_value_mgbm = get_shapley(best_mgbm2, train, X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAApwAAAFeCAYAAAAog8+HAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xm8XdP9//HXjaFKEmPQEhlQfjq37xZtDcU3368h9S31jTFC44tKUUXUGEOlhEYVFUNFEvNQGsTUisZX0Q8trVJFJmlCkGgGLUnO74+1Dttxh5Pr7tybm/fz8biPe85ee6+19j43vO9aa+/bUKlUMDMzMzMrS5f27oCZmZmZdW4OnGZmZmZWKgdOMzMzMyuVA6eZmZmZlcqB08zMzMxK5cBpZmZmZqVy4DQzMzOzUjlwmpmZmVmpHDjNzMzMrFQrt3cHzDqT8ePHV/r379/e3TAzM1tWGurZySOcZmZmZlYqB04zMzMzK5UDp5mZmZmVyoHTzMzMzErlwGlmZmZmpXLgNDMzM7NSOXCamZmZWakcOM3MzMysVA6cZmZmZlYqB04zMzMzK5UDp5mZmZmVyoHTzMzMzErlwGlmZmZmpXLgNDMzM7NSOXCamZmZWakcOM3MzMysVA2VSqW9+2DWaTRcsMj/oMzMrEOqHL9yGdU21LOTRzjNzMzMrFQOnGZmZmZWKgdOMzMzMyuVA6eZmZmZlcqB08zMzMxK5cBpZmZmZqXqlIFT0kRJp7Z3PzoSSRVJ32jvfiwNSc9KGtDe/QCQtEjSju3dDzMzs+VRKQ9kMpA0CDg1IjZr7750dJJ6A5OBnhHxSnV7RHx6GbQ9hfQ5jSu7LTMzsxVVpxzhtI5D0irt3QczMzNrX51+hFPS54CLgC8Cc4BfAsMjYnEuvwbYBVgLmA6cExHX57IdgQeBA4BzgfWA+4DvRsS8ZtrcFrgcWFXS/Lx5j4iYKGkH4HxgS2AmMDIiRrVwDrsD1wAbRcS7eVtXYBawe0Q8LOlcYF9gfeBV4OcRcVET9Q2iZvRV0mhgUUQMzu83AX4KfAOoAOOBHzZ33vm4YcD2wFPAQfn7rs1dZ+Dp/P1vkirAeRFxdu3oY2uuXT5uX+AMYGNgIXBvRBwsaTywCXCVpMuBRyOin6RuwCVAf2AecHpLbZiZmVnTOvUIp6Q1gQeAh4ANgd2BQ4HjCrs9AnyBFITOAkZL2qpQvhLQD/g88ClScD26uXYj4vfAEcDLEdE1f02U1Ae4F/gFsC4wCBguaZ8WTuVeYFHuf9U+pMD5u/z+r6Rw2A04LNf7ny3U2yhJqwG/zXX2AbYihbWf1VnF9qRA2BPYO29r7jp/Pn/fIl+rsxvpU6uunaTVgbHAURHRDegLXAUQEf2BacDg3G6/fNhFwOb5vD8H7En6OTAzM7NW6OwjnLsD75BG0yrAc5LOIwXOEQARcXVh/xslHQ/sSApbVSdFxHxgvqQ7ALWyP/sBT0XE6Pz+MUmjgMHALU0dFBGLJY0FDgHuyJsPAa7J50XNGsTfSrob2Jk0Iru09gAaIqI6sve2pNOARyUdVh0dbsbUiLgwv34n96+e69ycVl277F1gS0l/iog3gUlN7SipC2lEe/eImJW3DQW+XWc/zczMrEZnD5w9SeGnUtj2Ut5eDRfDgAGkEdAKsAbQo7D/4oiYXXi/gDSK2Nr+TK7Z9hJpBK0l1wDPSFo/t/81YP9qoaSjSSObGwMNwMeB6xuppx59gE0kza3ZXiFdpxktHD+1+KbO69ySVl27iFgoaTfSLxk/lvQycGFhOr9WD+BjwJTCttp2zczMbCl06il10lrBXpIaCtv65u2QRs0Gk6Z9146ItUjrCRv46JY00Z/eNduK/WlSRDwPPAkcSJpOfrB6R7ekrwPnAYcD6+XzGE/T5zGPFPiKPll4PRV4ISLWqvlaLSJaCpvw4XNv6To3dq1qfZRrNzEivkVag3sOME7Spk20/TppVLbYVm27ZmZmthQ6+wjn3aT1eCdLGkEauRsKVG806U5aGzkb6JJvpvk8cFcbtD0LWF9S94j4Z952A3CapIGk0ccvkULikXXWeQ1p/Wg34ITC9u7AYtJ5VPJNRrvS9FTzn3Lf9gDuIY0Sbg9Up+XvIo0Gngz8HJhPCqRfjYhf1dnXopau82xS8NsceKWxCmjltZO0AWlt64MR8VZh1La6LGBWbhd4b/nC9cCZkv4CvA38pP5TNTMzs1qdeoQzIt4i3fCzC+nO7fuAMaS7rwGuBR4HXiRNE29FM+v7ltJDpBuWJkuaK2mHiJgM7AYMAd4g3cxyWkTcXGedN5JG9boCdxa2V8/rCdII3XeAJoNhRLwEHANcAbwJ/BdwW6F8IbAT6Xo8D7wF/IZ0009rNHudI+Jt4DTghnytTmmkz629dl2Ao4ApkuYBlwIHR8SUXH4OcKCkOZIm5G3HkKbRnwf+TBotbmndqpmZmTWhoVKptLyXmdWl4YJF/gdlZmYdUuX4Uia261qG2KlHOM3MzMys/XX2NZylkbQdMKGJ4nMj4tylrO9k4OQmineNiLaa6v9I2vq828Lycu3MzMxWVJ5SN2tD48ePr/Tv37+9u2FmZraseErdzMzMzNqfA6eZmZmZlcqB08zMzMxK5cBpZmZmZqVy4DQzMzOzUjlwmpmZmVmpHDjNzMzMrFR+DqdZG/KftuxcSvozcGZmnYmfw2lmZmZm7c+B08zMzMxK5cBpZmZmZqVy4DQzMzOzUjlwmpmZmVmplqvAKWljSRVJvevYd5ikB5dBt1pF0uWSLmnvfnRkHekaSXpQ0rD27oeZmdnyaIV95kcOrZOBnhHxSh37DwJOjYjN2qL9iDiiLerpLCRNIV3fcdVty+IaSRoNLIqIwWW3ZWZmtqJarkY4bfkjaZX27oOZmZm1rw49wilpQ+AKYAfgVeD8mvLDgGOAnsDLwNCIuL+wS4OkkcBA4G3gkoj4SS57On//m6QKcF5EnN1EP7YFLgdWlTQ/b94jIiZK2iH3a0tgJjAyIkbVcW6jKYys5T4cBRyS63oWGBQRz+fyVYATgIOBTwKv5fO9VdLKwMnAIGBt4CngmIj4S6GtlYB3gb2ABcDxwHPAlbm9AA6IiH/kY1YHzgL2BtYEngCGRMSLLZzXjsCD+TzOBHoA3SQdAxwJbATMAa4jjWguljQe2AS4StLlwKMR0a+Ra9QLuBj4OunzvA34UUS83UKfvgj8HPgssBh4HtgdOAw4IO+zb959TWAJcBLp81gduJY6H2xrZmZmH9bRRzivIwWETYDtSYEKeC9sDiUFhrWBU4DbJRWnvLcnBdVPAHsCx0naP5d9Pn/fIiK6NhU2ASLi98ARwMt53645bPYB7gV+Aayb+zdc0j6tPN9BpIC3HjCdFJKqzgEOBPYBupNC+Au57ARSqN4N2BCYBDwgqXvh+O+QAto6wNmkoHkW8G1gA6BCCohV1SC6Ta7zceCuOkcsV8p9+WKuG+AVYNfc9z2BQ4HBABHRH5gGDM7Xtl9thTlU3w3MAnrlfn0duKCO/lwK3J/PfQPgOOCdiDif9DN2beFzXUy6zj/I/dwQeJ30s2RmZmat0GFHOCVtBOwEbBYRbwFvSTqTFBwgjWyeFRHVkcp7JD0E7EsKZ5BGHM+LiArwpKQrSKHu+jbq5n7AUxExOr9/TNIoUpC6pRX1jYiIafDeqOS4/LqBNNo2ICKeyfu+kr8gjSaeVxgNPSv3YXfghrzPbyPi7lw+hhSSx1bXr0q6lTTih6T1gP2BXhHxat52JnAssDXwSB3nMjR/bgBExG2Fsj9KGgvsDLQ4Gpx9Fdgc2DoiFgALJJ0K3CFpSP6Mm/IO6ZeWnhExBXishbYGAqMi4kkAScNJv3CYmZlZK3TYwAlsnL9PLWybXHjdB7hU0sWFbSvzfggDmFoTRKaQppTbSs+aPgG8RBoZa42ZhdcLgG75dQ9gDd4f0Wy2HxGxJN+E07OxuiNioaTa9hYW2uuTvz+T96tapabOpiwhjdC+R9J+pJHFvqTPaVVaDn5FPYHZOWxWvQSsRro+rzVz7CHAacAjkt4lBfkzI2JRE/tvTPpZAd67nlOb2NfMzMxa0JED54z8vRcpWAD0LpRPBc6IiOZGEntJaiiEzt68H0iXLGV/Gtt/OmnquKgvNWGrDcwmBcLNgb830Y/e1TeSuuT3re1HNVxtHhGzW3F8pRj0JfUkhby9gAkR8Y6kC4Bimm3p85gO9JC0ekQszNv6Av8iXZ8mRcRk0hQ+kj5LGiWfDPyyiXZn8MHr2UD6OTQzM7NW6LCBMyJekTQROF/SIcDHgdMLu4wEhkn6O+kGoNWALwOvV6eWSWs3T8g3Dn2GNGV8XC6bTQobm/PBUdGmzALWl9Q9Iv6Zt90AnCZpIGma/kvA4aSbY9pMRFQkXUa6FtNINxRtBKyTp9hHAydK+h1pZG4o6bO9u5XtvSbpeuAyScdGxAxJawHfBB6IiPktVFGrK2m98GzgXUnbAAeRblqqmkX6LJryBPAicKGkHwJrkdaiXtPCdDqSDs79/gcwF1hEWhtcbXcbSV0ioho+x5Ku9a+AP5NusNqw3pM1MzOzD+roNw3tD3yMNLo1CRhTLYiIK0l3h19Duut5GmnatHhTyyRS6JwF3AX8jLx+M9/ZfBpwg6S5kk5poS8PAQ8Ak/P+O+SRs92AIcAbpKByWkTc/FFOugmnADcDdwDzgIlA9QapEaTwez/pJqmdgH6FYNwahwF/AyZKmkcKXvuQbi5aKhHxHHAGcCcp8J3E+2tLq84BDpQ0R9KERupYBOxBmu6eRgqgj5PCYEt2Iq3hXQD8nvQzMDaXXUVarvBG/lxXIv2c/RwYT7qe6wO/q/uEzczM7AMaKpWlzg9m1oSGCxb5H1QnUjm+w04CmZl1FHU9NrCjj3CamZmZ2XLOv75nkrYDPjSVm50bEecuZX0nkx7G3phdI2LS0tTXkRQefl9rUkTsukw7Q9t/dmZmZta2PKVu1obGjx9f6d+/f3t3w8zMbFnxlLqZmZmZtT8HTjMzMzMrlQOnmZmZmZXKgdPMzMzMSuXAaWZmZmalcuA0MzMzs1I5cJqZmZlZqfwcTrM25D9t2Tn4T1qamdXNz+E0MzMzs/bnwGlmZmZmpXLgNDMzM7NSOXCamZmZWakcOK1dSNpR0qL27ke9JC2StGN798PMzGx55MC5ApE0SNKL7d2PjkTSFEkHtnc/zMzMOjMHzmVM0iorUrtmZmZmfthcG5A0Bfgl0A/4AvA8cGRE/EHSaGAV4F3gW8BNwJGSdgDOB7YEZgIjI2JUrm9H4EHgu8BZQDfg18CQiJif91k3H98PWA14CPh+RLxa06dvAl8BfgycAawqaX7u+h7AecCNETGycD5nAV+PiJ2bOecRwOYR8d+FbTsC44ENgQowDvgasDrwIjA0Ih5oor7RwKKIGFxzXU+NiHH5/XbAcGArYA5wGfDTiGj22ZeS9s3nvjGwELg3Ig6WNB7YBLhK0uXAoxHRT1I34BKgPzAPOL25+s3MzKx5HuFsO0cAxwDrALcC90jqnsv2ASYAPYAfSuoD3Av8AlgXGAQMl7RPob6VSIHnc8D/Az4F/BRAUgNwBynUfQboRQpG19f06TDgOFJgvSj38eWI6Jq/JgKjSMGWXHcX4BDgyhbO9xpgN0k9CtsOAW6OiAWkn63bgc3zOd4A3Fazf90kbQXcA4wgXcfdgSHAQS0ctzowFjgqIroBfYGrACKiPzANGJyvR7982EW531uRrv+epM/DzMzMWsEjnG3n6oh4EkDSecD3SCOIAI9ExE359UJJ+wFPRcTovO0xSaOAwcAthTqHRsRbwFuSTgfuknQE8CXgy8AuEfHv3OaJwOuSNo6IV/LxV0bEH/PrtyU11u8bgZGStomIx4D/BD4O/Kq5k42Iv0r6I3BgPr4b8B3SiCt5JHZc4ZARkoaSRlvvaa7uJnwPuCUi7szvn5d0CTAQGNPCse8CW0r6U0S8CUxqasccuA8Ado+IWXnbUODbreizmZmZ4cDZlqZUX0RERdI00hTuB8qynsDkmm0vkUbSiqbW1P8xYD2gT379ak2I/BdpiviVwjHNioiFksaRwu5j+fuYapBtwTXAkcBI4H+AVyLi/wAkfZw0Grlb7vMS0khrq0Y4See8k6S9Ctu6ANObOyif326kkd4fS3oZuDAiakeDq3qQru2Uwrbaz8rMzMyWggNn2+ldfZGnvKvBbytS2CqaTgpiRX35cHjqRQqi1fr/DbxOCqILgHUiorbuotqypvYdBfyfpB+TpvG/0EydRdXR0S+RlgVcUyg7Dtge2BmYkkP46zT9N1fnkYIpAJJWBtYvlE8FfhkRR9XZt/fkpQMTJa1EWkd7m6THI+IlPnxNXgfeIV3v4rU3MzOzVvIazrZzqKQv5bvBTyDdKHN3E/veAHxZ0kBJK0v6KnA4cHXNfsMldZe0PjAMGJsDZgBPAxfnm4eQ1CPfHNOcWcD6hbWlAETEM8CzwG3AExHx13pOOCLmkqbezwG2Aa4tFHcnBeQ3SDcqnQ6s1Ux1TwI7S+oj6WOkm5yKd9ZfBuwrqb+kVfJ12yrffNUkSRtI2lvSmhGxGJibixbn77NI6zWr57SYtBb2zHxsd+AnzbVhZmZmzXPgbDtXABeT7p4eQFoD+FZjO0bEZNII5xBSIBsLnBYRNxd2W0wKrH8G/ga8TBo1JIfOPUmjhU9KmkeaDt+xhT4+BDwATJY0tyasjQK+SMs3C9W6BtgVuC8iZha2/5QU7v5BGilcSPNT/NeR7sR/Ku8/DZhRLYyIv5DWxB5Luqv/NWA0LU/RdwGOAqbk63QpcHBEVPtyDnCgpDmSJuRtx5Cm0Z8nXf/xvB9QzczMbCk1VCrNPlHG6lD7+J42qG9H4MGIWGZLHnKbdwCfjIiFy6rdzqbhgkX+B9UJVI73aiMzszo1tVTuAzzCaUhaDTiedFe7w6aZmZm1Kf8av4LLd32PI01l/7imbAKwXWPHRUTX8ntXH0knAyc3UbxrRDT5GCQzMzMrn6fUzdqQp9Q7B0+pm5nVra4pdf9X1awN/XqLCfTv37+9u2FmZtaheA2nmZmZmZXKgdPMzMzMSuXAaWZmZmalcuA0MzMzs1I5cJqZmZlZqRw4zczMzKxUDpxmZmZmVio/+N2sDfnB7x2bH+huZtbm/LfUzczMzKz9OXCamZmZWakcOM3MzMysVA6cZmZmZlYqB04zMzMzK5UDZzuSNFHSqe3dD2uZpBclDWrvfpiZmS2PHDhtuSNpR0mL2qguh34zM7OSOXCamZmZWan8FOQOQtLngIuALwJzgF8CwyNicS6/BtgFWAuYDpwTEdfnsh2BB4EDgHOB9YD7gO9GxLwW2v08cDHwaWAl4DFgSES8lMtH5+3vAnsBC4DjgeeAK4EtgQAOiIh/5GPWBUYC/XIz9wE/iIg3c/kU4NSIGJff9wYmAz0j4pVCm/8C9sltnhURoyR9EpgArCRpfq7/qIi4tplz3AUYAWwKvAP8KSJ2kXQJsB2wraSTgBkRsYWkVYDzgAOBJflczMzMrJU8wtkBSFoTeAB4CNgQ2B04FDiusNsjwBdIgfMsYLSkrQrlK5EC3ueBT5GC69F1NF8BhgEbAb2B+cC4mn2+A9wGrAOcTQqaZwHfBjbIdZxZ2P86YG3g/+Wv9YCxdfSlts3xuc3vA5dI6pVD7a7A4ojomr+aDJvZGFKoXjOf5zkAETEEmAScnevZIu9/ErAH8DWgD+m69FrK/puZmVnmEc6OYXfSyNs5EVEBnpN0HilwjgCIiKsL+98o6XhgR+Cvhe0nRcR8YL6kOwC11HBEPFN4+29JZwJ/lrR6RCzM238bEXcDSBoD/AIYGxGv5G23Aofl158E/hP4VETMyduOA56X9ImImFnnNfltRPw6v75d0lxS4J5a5/FF75BGNzeIiFnAxBb2Hwj8JCJeBMjX+rutaNfMzMxw4OwoegJTc9iseilvR1IX0ijkANIIaAVYA+hR2H9xRMwuvF8AdGupYUmbkkLt1nn/ah968H64ey8kRsRCSR/YBiwstNUzf59ccy7VsnoDZ+1+dZ1PE/YETiYF6dnAFRFxUTP7bwxMqb6JiAWSXmtl22ZmZis8T6l3DNOBXpIaCtv65u0A+wGDgb2BtSNiLeBpoIGP7nJgHvC5iOgOfD1vb23d1T73LmzrW1M2jxSYqz65lG0sWZqdI+LpiBgArA8cDgyXtFMzdc2g0H9JteHezMzMloIDZ8dwN/Ax4GRJq0raAhgKVKfRuwOLgNlAF0mHktZqtoXupNHDuZLWI63NbLW8xvJ+4EJJa0laG7gQmFCYTn8S2E9SV0k9gNOWsplZpJuG+rS0Y76eB0taL48gzyGFzMWFujarOWwscIKkTSV9HDgf/1sxMzNrNf9PtAOIiLdIN/zsArxKuqt7DPDTvMu1wOPAi6TRt61IN7u0hR+Q7tT+Z67zrjao80DSKObfgOeBuaR1kVWnkgLfTNJ6yhuXpvKIeIG0jvQJSXMlHdTCIQNIa0jnA78GzoiIh3PZSEC5nmfztuGkz+Ax0tKAabRu7aiZmZkBDZVKpeW9zKwuDRcs8j+oDqxyvJetm5m1sbqW4HmE08zMzMxK5V/3OzlJ25EelN6YcyPi3GXZnzJIOgAY1UTx4RFx3bLsj5mZmX2Qp9TN2tD48eMr/fv3b+9umJmZLSueUjczMzOz9ufAaWZmZmalcuA0MzMzs1I5cJqZmZlZqRw4zczMzKxUDpxmZmZmVioHTjMzMzMrlZ/DadaG2vtPW/pPN5qZ2TLm53CamZmZWftz4DQzMzOzUjlwmpmZmVmpHDjNzMzMrFQOnGZmZmZWKgfOAkkTJJ3Y3v1YEXzUay3pQElT2rBLzbV1laTRy6ItMzOzzsjPUCmIiF3r3VfSRODBiDinvB4tPUmDgFMjYrP27ktzluZal0XSMOAbEbFLe/fFzMysM/MIZ4kkrdKR62sPneEczMzMbOl4hLOgOmoJjAMmAwOBHwE9gd8DB0fETEmXANsB20o6CZgREVvkaddVgHeBbwE3AUdK2gE4H9gSmAmMjIhRdfbnT0BvYCfgXEnjgKuALwOrAs8Ax0bEk5K2BS4HVpU0P1ezR0RMlPQZ4ELgS8DbwHXA6RHxbgt9qJ7TEmBPYDZwdkSMLuyzHTAc2AqYA1wG/DQiKpJ2zNf0EOBMoAfQrXaEWNLngIuAL+Y6fgkMj4jFufyrud4t8zW5v6Xrl4/rDYwCtgYqpM91P+ALwMlAl8K1+lxEvCzpUOCU3Nc7SQ+1XVRPe2ZmZvZhHuFs3gBge2AjYA3gLICIGAJMIgWvrhGxReGYfYAJpLDyQ0l9gHuBXwDrAoOA4ZL2qbMPhwIXA2vm711IwasXsCHwFHC7pFUi4vfAEcDLuV9dc9hcH3gYuD2fy7bAf5DCdD3+B7gPWAc4HPiFpK8BSNoKuAcYkc95d2AIcFDh+JWA3UhhcoPayiWtCTwAPJTPafd83scVyicAt+Y+/AD4Xp19PxeYlttdj3T950TETblsYuFavZzD86Wk67hO7teAOtsyMzOzRniEs3lnRsTrAJKuBwbXccwjOcwALJS0H/BUYUTwMUmjcl231FHfrRHx22p9pPA0rVoo6VTgaGBz4K9N1DEQeLowqjpD0nDgPHKIbsFjETEuv35Q0m2k4PYoKfjdEhF35vLn8wjwQGBMoY6hEfFWE/XvDrwDnBMRFeA5SeeRAucIYA9gAXBeLv+DpKuBA+ro+zukENs3Ip4jjQg3ZyDpmj+Q34+RdHgd7ZiZmVkTHDibN7PwegHQrY5jptS870maxi16iTQ9XY8P1CdpPeCnwI7AWqSpbkiji03pA3xd0tzCtgbSyONS9yG//1Kh7p0k7VUo7wJML7xfUvO+Vk9gag6TVS/l7QAbN1Jee02bcgJwGjBe0hqkUdIfRcT8JvbfGIiabfW2ZWZmZo1w4Gy9JXVun06aTi7qS/MBrLn6hgOfALbO60m7Af8kBcim+jWVtF5y9zrbrNW7kfevFOr+ZUQc1czxlZqwWGs60EtSQ2G/4jWa0Uh5bZ8aFRGzSSPAR0vqS1qTeSJwOo1fqxmN1N0beLGe9szMzOzDHDhbbxZQz6OHbgBOkzQQuJ40Mng4cGQr2+1OmlqfI6kraVq8tl/rS+oeEf/M28aQ1pMemvvwDilEfSoi7q2jzW3y0oCbgR2AvYHqo4QuAx6WdC9prWoF+BTQIyIervOc7ibdMHSypBGkUdOhpJt9AO4irV89QdJI4LPAd4F/t1SxpAHAE6RR2bdI5744F88CNpG0akS8k7eNBe7NN0s9DOxLuuHIgdPMzKyVfNNQ640EJGmupGeb2ikiJpNGOIcAb5ACzWkRcXMr2z0dWD/X9QxpHeXiQvlDpBtdJue+7RARs4BvAv9NCl5zgF+RRhHrcXM+hznA1cBREfF/+fz+QlpjeSxpCcJrwGian+L/gLy2sx8pxL5KukFpDGnpABExl7TOc0Duw8Wkm7Dq8UVScJwPPEu6yWpELruFNIo6K1+rPjkkf5/0JIA3gf8iPW3AzMzMWqmhUmluptNWdHmkb1FE1HPD1Aqv4YJF7foPqnK8Jy3MzGyZamh5F49wmpmZmVnJPBzSjiSdTHr4eGN2jYhJy6APB/D+WslaHf5xQJI2oenHQY2LiCOWZX/MzMzswzylbtaGxo8fX+nfv397d8PMzGxZ8ZS6mZmZmbU/B04zMzMzK5UDp5mZmZmVyoHTzMzMzErlwGlmZmZmpXLgNDMzM7NSOXCamZmZWan8HE6zNlTPn7b0n580M7NOxM/hNDMzM7P258BpZmZmZqVy4DQzMzOzUjlwmpmZmVmpHDjNzMzMrFQOnJmk+ZK2be9+rCg+6vWWdJWk0W3YpebaelHSoGXRlpmZWWfk57NkEdF1afaXVAG2i4hHSupSq+QQtigiBrd3X5qztNe7DJImAg9GxDnt3RczM7POzCOcJZK0ShvW1SBpuf8FoS2viZmZmS0flrsAI2kKcAWwM7A1MAX434h4NAeyk4FBwNrAU8AxEfGXOup9b8QyT5+eClwMnAisAdwMfC8iFkt6Oh+MO6gHAAAgAElEQVR2v6QlwI0RMTj37ZfAN4GvAIOBGyUdCRwLbAg8B5wQEZPq7NOxwEHAp4FvSloDOBf4FLAI+A1wdES8JulE4IB87L65mjVzn/8bOA3YFJgJnBMR19XRh+o59QO+ADwPHBkRfyjscxhwDNATeBkYGhH357JhwPakz+Kg/H3X2hFiSXsDpwO9SZ/psIj4VaGNQ4FTgB7AnaQHzS6qo/+7ACPyeb8D/CkidpF0CbAdsK2kk4AZEbFFDsTnAQcCS4CRLbVhZmZmzVteRzgPBY4G1gQeAK7N208ABgK7kcLdJOABSd1b0UYvYANSUPkKsA+wL0BEfD7v0y8iutZMXx8GHAd0A+6UtB9wdu7XusCVwL2SetXZj+8CA4CuwB+BfwNDSMHrs8AngZ/lfp0PXAdcm/vVNYfN/wCuJoXXdYCDgUskbV9nH44gBcp1gFuBe6rXNIfNoaSguzYpFN4uabPC8duTQm5PYO/ayiV9Lff7JNI1Ohm4QdLWuXw74NLcj3VIn/mAOvs+hvSLw5rARsA5ABExhPTzcXa+Tlvk/U8C9gC+BvQhBeB6PyszMzNrxHI3wpmNiohnId08AhwraU3gEOC8iHg+l51FGmXcHbhhKdt4Gzg9IhYDL0r6DSBSMGrOlRHxx2odkg7J/X08b7ta0mBgf2B4Hf24ICJeyq8XA8U1o7MknU8agWzOMcDPCqOqT0gaRwrBv6ujD1dHxJMAks4DvkcKZdfnus+KiOqo7z2SHiKF8+rayKkRcWF+/U4j9Q8CbouICfn93ZJ+RfrF4vHcz1sj4oFcPkbS4XX0u9repsAGETELmNjC/gOBn0TEi/l8jyeFfjMzM2ul5TVwziy8XpC/dyONoE2uFkTEkjwl3LMVbbyWw2axnW51HDel5n1P0nR80UtL0acP1Cfpy6Qp9c8Dq5Omllu6AacPaTr+uMK2lUgjfEvVh4ioSJoGbFyo+1JJFxf2Xxl4pfB+agv19wSerNn2EvCl/HpjIGrKJ1OfPUkjpn+WNBu4IiIuamb/jfng+S6Q9FqdbZmZmVkjltfA2ZTppClQACR1ye+nl9BWpYntS5rrU9YXGF9nO7X13Uia1t4nIv4paY+aumr3hxT4RkfEiDrbrNW7+kJSA7AJ7wfKqcAZEXFLM8c31qeipq5R9XOb0Uh5b+DFFuolj7wOyP3+Bmnd7TMR8dsm+vWBtvKa2R4ttWNmZmZN62yBczRwoqTfkUaphpLO8e4S2poFbM4Hp7ib6tPPJP2a92+c+QKwXyvb7Q68BcyTtAlpzWFtv7aR1CUiqoHqImC0pMeAR0mjm58FGiKiduSwMYfmKe4/Az8gjaxWr+lIYJikvwNPA6sBXwZery5tqMO1wIOSxgIPkm5Q2gvYMZePJa17HQ08TJqu35oWAqekVUnX+e6IeF3SHFLIrI5czwI2qzlsLHBCfmTSP4DzWX7XOpuZmXUIne1/pCNIazXvB14FdiLd2PPPEto6BThL0hxJo5raKSKuB84ExgFvAEcCu0VES9PMTflf0rrUecDtQO3I4lWku+rfkDRX0kr5jvHDSNfnddKShJG0PBVfdQXpxps5pJt1do+It/L5XUkKZdfk8mmku+HrfvxRRPwf6UamC3Id5wMHRsRjufxh4Pv53N4E/gu4qc7qBwDPS5oP/Jo0GvtwLhsJKF+nZ/O24cB9wGOkaftptLwkwMzMzJrRUKk0NTNs9t5jkU6NiHHt3ZflQcMFi1r8B1U5vrNNLJiZ2QqsoZ6dOtsIp5mZmZl1MCvMUEueMm3seYpTI+LTy7o/AJImkB4+/iHL6k8/Srqc9JDzxmy1LPrwUUg6AGhqScPh9Tzc3szMzMrlKXWzNjR+/PhK//7927sbZmZmy4qn1M3MzMys/TlwmpmZmVmpHDjNzMzMrFQOnGZmZmZWKgdOMzMzMyuVA6eZmZmZlcqB08zMzMxK5cBpZmZmZqXyg9/N2lBzf0vdf0PdzMw6IT/43czMzMzanwOnmZmZmZXKgdPMzMzMSuXAaWZmZmalcuA0MzMzs1I5cNp7JE2QdGJ796OjkXSVpNHt3Q8zM7PllZ/TYu+JiF3buw/LkqRhwDciYpf27ouZmVln5hFOMzMzMyuVRzjtPZImAg8C44DJwEDgR0BP4PfAwRExM+/bFRgG7AX0AKYDh0fEJEmrA8Nz2ceBR4CjI2JaoZ2ngD7ALsBrwP+SHh57EbAJ8BtgYETMy8esC5wP9ANWAx4Cvh8Rr7ZwTr2BUcDWQCWf137AF4CTgS6S5ufdPxcRL0s6FDgln9eduV+LluJSmpmZWYFHOK05A4DtgY2ANYCzCmVXk0LczkB34FvAzFw2Etgmf/UCXgfGS1qpcPxBwE+AtYCbgLGk0Lk90BvYAjgaQFIDcAcpMH4m1zkPuL6OczgXmAZsAKwHDALmRMRNuWxiRHTNXy9L2g64FDgCWAd4IF8HMzMzayWPcFpzzoyI1wEkXQ8Mzq/XB/4H+ExETM77vpjLugAHA/0jYkbedizwJvBV0kgpwM0R8XguH0caSR0REW/mbXcByvt+OX/tEhH/zuUnAq9L2jgiXmnmHN4BNgT6RsRzwDMtnPNA4NaIeCC/HyPp8BaOMTMzs2Z4hNOaM7PwegHQLb/unb+/0MgxPYCPkaauAYiI+aRp855N1L2wiW3V9vrkOl+VNFfSXOAl4F+k6ffmnJD7Ml7STEk/z8sBmrIxMKVm2+RG9jMzM7M6eYTTWmNK/r458NeastnAv0mhtDrq2RVYn7TOszWmkgLvOhGxZGkOjIjZpKn5oyX1Ja3JPBE4HWisrhm8H6irepPPxczMzJaeA6cttYh4TdKtwGWSBpEC4aa57EVJY4CzJf0VmAtcCDwPPNHaJoGngYslnRERb0jqAewcETc2d6CkAbndKcBbpCn2xbl4FrCJpFUj4p28bSxwb37u5sPAvqS1qg6cZmZmreQpdWutQ4E/kULZPNLI4Ya57AekkPgH0g07nwC+FRGLG6mnRXlUc0/S3eJPSpoHPAbsWMfhX8x9nA88S7o7fkQuu4U06jorT9X3iYiHge8DV5HWnf4X6aYmMzMza6WGSqXS3n0w6zQaLljU5D+oyvGeUDAzs06noZ6dPMJpZmZmZqXykIst1yRtwodvXKoaFxFHLMv+mJmZ2Yd5St2sDY0fP77Sv3//9u6GmZnZsuIpdTMzMzNrfw6cZmZmZlYqB04zMzMzK5UDp5mZmZmVyoHTzMzMzErlwGlmZmZmpXLgNDMzM7NS+TmcZm2oqT9t6T9raWZmnZSfw2lmZmZm7c+B08zMzMxK5cBpZmZmZqVy4DQzMzOzUvlOBltmJJ0E/ABYA/hmRPxhGbV7ObAoIoYsi/bMzMzsg3yXui0TkjYGpgGfiYi/tnd/yuK71M3MbAXju9StQ+kNLPkoYVPSKm3XHTMzM1tWPMJpHyCpKzAM2AvoAUwHDgeeBIbn7R8HHgGOjohp+biJeZ/eQD/gNeC4iLhT0gBgNLAasAB4NSI2lbQuMDLvD3Af8IOIeDPXOQX4JfBN4CvAYGBLYDsggENJvzT9GLgNuCbv9wJwYEQ8l+sZTZpSH5zfV4CjgENyfc8CgyLi+VzeDbgU2AOYB5yW+7FLRExs7vp5hNPMzFYwHuG0Vrka2BrYGegOfAuYSQqG2+SvXsDrwHhJKxWOPRi4EFgTuAS4VtLqEXETsCuwOCK6RsSmef/rgLWB/5e/1gPG1vTnMOA4oBtwZ962PfB3YEPgQGBE7vdRwDrAc8DFLZznIGDv3OZ04OeFsp8BfUlh9LPA7sBKmJmZWat42MXeI2l94H9I6ywn580vSupCCpP9I2JG3vdY4E3gq8Dv8743RcSjufwK4KfA5sDTjbT1SeA/gU9FxJy87TjgeUmfiIiZedcrI+KP+fXbkgBeiIir8rYJkt4A7iuMaF5PCrPNGVEYnR0NjMuvVwIOAHaNiNfytpPzdTEzM7NW8AinFfXO31+o2d4D+BhQDaFExHzStHnPwn4zC+UL8stuTbRVPW5yYdtLNWUAUxo5dmbN+4U12xY2025jdSwo7L8esCowtVBefG1mZmZLyYHTiqbk75vXbJ8N/Jv3A2l1ref6pOno1qge17uwrW9NGcCSVtbfWq8D75CWDVRtsoz7YGZm1ql4St3eExGvSboVuEzSINLIXnW95RjgbEl/BeaS1mo+DzzRyrb+Iel+4EJJB5MWHV8ITChMpy9zEbE4T8kPk/Rn4F+km5LMzMyslTzCabUOBf4EPEy6Q/tO0s05PyDdGf4H0vM0PwF8KyIWf4S2Dsxt/I0UXucCAz9CfW3lGNI5vgD8BXgAqJBGec3MzGwp+bFIZi2QtAUpEG8UEf9obl8/FsnMzFYwdT0Wyf8XNKshqS9pVPdx0k1EI4HftRQ2zczMrHGeUjf7sNWAK4C3gD+T7nrfv117ZGZmthzzCKdZjfznNz/T3v0wMzPrLBw4zdrQr7eYQP/+/du7G2ZmZh2Kp9TNzMzMrFQOnGZmZmZWKgdOMzMzMyuVA6eZmZmZlcqB08zMzMxK5cBpZmZmZqVy4DQzMzOzUjlwmrWhb/1t1/bugpmZWYfjwGlmZmZmpXLgNDMzM7NSOXCamZmZWakcOM3MzMysVK0KnJI2llSR1LuOfYdJerA17SwLki6XdEl796Mja89r1BZtSzpA0tNt1SczMzNbOiu3dwdyaJ0M9IyIV+rYfxBwakRs1hbtR8QRbVFPZyFpCun6jqtua89r1BZtR8R1wHVt0B0zMzNrBU+pryAkrdLefTAzM7MVU10jnJI2BK4AdgBeBc6vKT8MOAboCbwMDI2I+wu7NEgaCQwE3gYuiYif5LLqVOffJFWA8yLi7Cb6sS1wObCqpPl58x4RMVHSDrlfWwIzgZERMaqOcxsNLIqIwfl9BTgKOCTX9SwwKCKez+WrACcABwOfBF7L53urpJWBk4FBwNrAU8AxEfGXQlsrAe8CewELgOOB54Arc3sBHBAR/8jHrA6cBewNrAk8AQyJiBdbOK8dgQfzeZwJ9AC6SToGOBLYCJhDGvk7NSIWSxoPbAJcJely4NGI6NfINeoFXAx8nfR53gb8KCLebqFPw4Dt8jkeSvqF58f5+GuArwAvAAdGxHOFa7YoIgZLagDOyefUDXgDuDAifi5pbdLP6E6kn+tXgCMiYlLtqLikicCTQG+gH+kzPC4i7szlDcCPgO8BqwPXAp8DJkXEsObO0czMzD6s3hHO64DFpDCyPSlQAe+FzaHAAaSQdQpwu6TilPf2pKD6CWBP4DhJ++eyz+fvW0RE16bCJkBE/B44Ang579s1h80+wL3AL4B1c/+GS9qnzvOrNYgU8NYDpgM/L5SdAxwI7AN0J4XwF3LZCaRQvRuwITAJeEBS98Lx3yEFrHWAs0lB8yzg28AGQIUUEKuqQXSbXOfjwF11jliulPvyxVw3pCC2a+77nqTgNxggIvoD04DB+dr2q60wh+q7gVlAr9yvrwMX1NEfSD8Lf8/nciAwAriaFPLXIYXvi5s49j9IQX/riOgGfBV4JJedQAqHvYC1SNezuSUaBwMXkkL8JcC1OdwDHET6Bao/6brNzP02MzOzVmhxhFPSRqRRo80i4i3gLUlnAtURzGOAsyKiOlJ5j6SHgH1J4QzS/7DPi4gK8KSkK0ih7vo2Oo/9gKciYnR+/5ikUaQgdUsr6hsREdPgvRG2cfl1AykYDYiIZ/K+r/B+sDmEdJ7V0dCzch92B27I+/w2Iu7O5WNIIXlsdf2qpFuBw/Lr9YD9gV4R8WrediZwLLA174et5gzNnxsAEXFboeyPksYCOwMtjgZnXwU2J4W+BcACSacCd0gakj/j5rwQEVfl1xMkvQHcVxjRvJ6m11u+A6wGfFrS7Ih4jTQ6WS1bF9gC+GNEvNBEHVU3RcSjuc0rgJ/m83qa9EvDqIj4Yy4fQfrczczMrBXqGeHcOH+fWtg2ufC6D3CppLnVL+CbpCnbqqk1QWRKod620LOmTwAv5e2tMbPwegFp+hbStPQavD+i2Ww/ImIJ6VyL/ZhZKF/YSHsLC+31yd+fKVzbN4FVqO/clpBGaN8jaT9Jf5D0hqS3SEGqRx11VfUEZuewWfUSKQjWU8/MmvcLafr8PyAiJpKWLJwKvCbpfknKxSOA35Cmv2dLulbSBo3VU9uPwrlU292Iws97/tn9wHU0MzOz+tWzhnNG/t6LFCwgrX2rmgqcERHNjST2ktRQCJ29eX9UcEl9XX1PY/tPJ00dF/Wl7UPCbFIg2pw0LdxYP3pX30jqkt+3th/V0LN5RMxuxfGVYtCX1JM0WrsXMCEi3pF0AaDCMS19HtOBHpJWLwTmvsC/SNenVBFxBXBFnv4eBtwObJJD4ynAKXnN8ThSCB3YimZmkH7egfdGtlv7y4uZmdkKr8XAGRGv5Jsszpd0CPBx4PTCLiOBYZL+TpqOXA34MvB6dWqZtHbzhHzj0GdIU8bH5bLZpJCzOc2vuauaBawvqXtE/DNvuwE4TdJA0jT9l4DDSTfHtJmIqEi6jHQtppFuKNoIWCdPsY8GTpT0O9LI5lDSNb67le29lqeYL5N0bETMkLQWaQT5gYiY30IVtbqSRrVnA+9K2oa0XvG5wj6zSJ9FU54AXgQulPRD0nrJs4Fr6phO/0gkfRX4WO7Dv4F5pLXFSOqf+/UCMJ8UgBe3sqmxwHmSbid9xkeTbhAzMzOzVqj3pqH9Sf+jn066EWZMtSAiriTdHX4N6a7nacBppGnfqkmk0DkLuAv4GXn9Zr6z+TTghjxtfEoLfXkIeACYnPffISImk0Y4h5DuXB4LnBYRN9d5fkvjFOBm4A5S4JkIVG+QGkEKv/eTbpLaCehXCMatcRjwN2CipHnAn0k3LC11uMvrJM8A7gTmAifx/trSqnOAAyXNkTShkToWAXuQlkRMI4W/x0l325etK+ln53XS59wPGJDLNgXGA/8khf23SYG/NcYAlwL3kD7HjYHHSCHXzMzMllJDpVLqoJTZci8vjZgOnBARzd7o1nDBokrl+Hb/ewpmZmbLSkM9O/n/jGaNkLQvaRS7C+mZnKsDHxrxNTMzs5Z1uMApaTua/h/7uRFx7lLWdzLpzubG7BoRk5amvo6k8PD7WpMiYtdl2hna/rNrZ0N4/1FRfwF2i4g57dgfMzOz5Zan1M3akKfUzcxsBVPXlLr/lrpZG/r1Fp51NzMzq+XAaWZmZmalcuA0MzMzs1I5cJqZmZlZqRw4zczMzKxUDpxmZmZmVioHTjMzMzMrlQOnmZmZmZXKgdPMzMzMSuXAaWZmZmalcuA0MzMzs1I5cJqZmZlZqRw4zczMzKxUDpxmZmZmVioHTjMzMzMr1crt3QFbfkmaCDwYEee0d1+qJP0Y2B9YF/gX8DvguIiYVthnU+ACYKe86Tlgu4h4N5cLuAz4DDATOCMixi2zkzAzM+tkPMJp7UbSKiVUOxb4QkR0B3oD04AbC232ACYBTwObAOsAQ4DFuXxNYAJwG7A2cARwuaRtS+irmZnZCsEjnNYqki4BtgO2lXQSMCMitpB0GHAM0BN4GRgaEffnY4YB2wNPAQcBT0k6EpgMDAKGAr2Ah4ED8vtDgSXA2RFxaUv9iojnC28b8rFbFLYdB0yLiGHFwwqv9wIWAudHRAV4QNKvgP8Fft9S+2ZmZvZhHuG0VomIIaSRwrMjomshbA4lhcW1gVOA2yVtVjh0e9I0dU9g78L2vYFvkEYdewOPAy8BnwQOAS6StEk9fZO0v6S34P+3d+/hchRlHse/L7kAIQRYkygSQhIBlYUgpPDBlUtQboqXiEREAmIUE++XR1Y3gptVlIsgKy4sEDBBUBDYJZqAREB4FB7FLblfAyEnIYGYQC4mRAmB2j+qJqnTmZkzczIz50zy+zzPPGe6q7u6urq6+53q7tOsIQa/U7PkI4DnnHO3OOeWO+ceds6dnKXvDzyQgs2S+9N4ERER6Qb1cEojfQX4rvf+oTR8q3PuLuDjQOk+zwXe+wvT93XxdkkgBq7LAZxzs4HjvPfTUtpvnHMrgAOIl8ir8t7/AviFc+5NwKeBR7LkwcBBwInAh4kB6Czn3ALv/T3AjsCqQpYrgUFdrr2IiIiUpYBTGmkkcIlz7uJsXF9gUTa8oMK8L2Tf1xaGS+N2rKcw3vslzrlpwLPOueEpoF0N/NF7f1Oa7Hbn3G3Ah4B7UvqIQlY7A3+rZ9kiIiKykQJO2RyvF4YXEJ/ovrGOeZqtL7AD8dL8cuBBYM8y05UuoT8EjCukHZDGi4iISDco4JTNsYTOwdtFwFTn3NPEAG07YAzwYuFhnqZwzm0DfB64wXu/1Dk3DPgJ0AGUln858Afn3Djg18DhwNHAeSn9ZuB859wZwI+JD0YdDxzV7PKLiIhsqfTQkGyOi4j/tnKlc+6xdM/l+cB0YAXxfsuzgGb8+6NK3g886px7mfjg0VrgSO/9egDv/Z+I/6fzPOLl858An/Te/zGlr0x5jCfeyzkNmFxKFxERkfpZCKHrqUSkJrNmzQof/OAHe7oYIiIirWK1TKQeThERERFpKt3DKW3FOXcZMKFC8j75KyxFRESkd1DAKW3Fez+Z+LpJERERaRO6pC4iIiIiTaWAU0RERESaSgGniIiIiDSVAk4RERERaSoFnCIiIiLSVAo4RURERKSpFHCKiIiISFMp4BQRERGRplLAKSIiIiJNpYBTRERERJpKAaeIiIiINJUCThERERFpKgWcIiIiItJUCjhFREREpKkUcIqIiIhIUyngFBEREZGmUsApIiIiIk1lIYSeLoPIFmPbbbd9dN26df/o6XJsKfr27Tt4/fr1L/Z0ObYUqs/GUn02luqzsVpYny+GEI7tsjwtKIjIVmO//fb7h/fe9XQ5thTOOa/6bBzVZ2OpPhtL9dlYva0+dUldRERERJpKAaeIiIiINJUCTpHGuqKnC7CFUX02luqzsVSfjaX6bKxeVZ96aEhEREREmko9nCIiIiLSVAo4RURERKSp9G+RROrknNsbuBp4A/AScKr3/unCNH2Ai4FjgQCc672/stVlbQc11udZwMeB14BXgSne+zmtLms7qKU+s2nfCjwAXOq9/0brStk+aq1P59zHgLMAI+7zR3rv/9rKsraDGvf3ocB0YHegH3AX8GXv/foWF7fXc85dAHwUGAHs571/tMw0veJ8pB5OkfpdBlzivd8buAS4vMw0JwN7AnsB7wKmOudGtKyE7aWW+vwzcJD3fjQwEfilc277FpaxndRSn6WT0OXAzBaWrR11WZ/OOQdMBY7y3u8LHAKsamUh20gt7XMK8ETa30cDY4DjW1fEtjITOAxYUGWaXnE+UsApUof0y/tA4Lo06jrgQOfckMKkJwLTvPeve++XEQ8K41tX0vZQa3167+d479emwYeJvUhvaFlB20Qd7RPgW8BsYG6Litd26qjPrwEXeO+XAHjvV3nv9caxgjrqMwA7Oue2AbYF+gOLW1bQNuK9v8d7/1wXk/WK85ECTpH67A4s9t6/BpD+Pp/G54bT+RfnwjLTSO31mTsVmOe9X9SC8rWbmurTObc/cAxwUctL2F5qbZ/7AKOcc793zt3vnDvTOWctLms7qLU+vwfsDbwALAHmeO/vbWVBtzC94nykgFNE2oZz7nDiyeikni5Lu3LO9SP+f77JpRO/bLY+xEu/RwGHA+8DTunRErW38cQrGbsCuwGHOedO6NkiyeZSwClSn+eA3dL9b6X74N6cxucWAntkw8PLTCO11yfOuXcB1wLjvPdPtbSU7aOW+twVeAtwq3OuA/gqcLpzrlf9k+heop79/Sbv/Sve+9XAr4B3trSk7aHW+vwS8PN0CXgVsT6PaGlJtyy94nykgFOkDt77pcCDbOxhOwl4IN0Xk7uReBLfJt2fNA64qXUlbQ+11qdz7iDgl8AJ3vv7W1vK9lFLfXrvF3rvB3vvR3jvRwD/Sby/67MtL3AvV8f+/gvgaOecpR7k9wIPta6k7aGO+pxPfKIa51x/4Ehgk6evpWa94nykgFOkfpOBLznn5hJ/iU8GcM7dmp5WBbgGeBZ4GvgT8F3v/fyeKGwbqKU+LwW2By53zj2YPvv1THF7vVrqU2pXS31eDywFHicGVI8BV/VAWdtBLfX5VeBQ59wjxPqcC0zricL2ds65i51zi4BhwB3OucfS+F53PtKrLUVERESkqdTDKSIiIiJNpYBTRERERJpKAaeIiIiINJUCThERERFpKgWcIiIiItJUCjhFtmJmdoyZ/SEbHmtmHT1YpJYxsxlmdmUD8xthZiEbHmJmC8xscA3zTjazaxpVlnZgZoea2cqeLsfWyMwm1LOfN3pfkeqatW90Y7ufa2bfa9TyFXCKbKXMzIjv0v73Lqb7nJk9amZ/M7MVZubN7MQsvcPMJpSZb5PxFs1NeQ0spI01s2Bma9LneTObbmb/tHlr2jNCCMuI/xC8q/rdAfguMLUFxeo1Qgh/CCHs3NPlqMTMpprZHT1djq1Bs+razO42szMbnW+zFfeNHmyL5wFfMLPdGpGZAk6RrdfRQH/grkoTmNlJxIDp08BOxNfQfQ1Y0c1lHgGMAl6n/PvQXwshDAwhDAQOAd5FfBNOu/op8CkzG1RlmgnAIyGEeS0qUydm1sfMdC4QkU5CCCuA3wCTGpGfDjIiLZB6+840s7tS790jZjbazE4ys2fMbJWZXWlmfbN5hpvZTWa2xMxeMLMrzGzHLP0HZvZsym+emX01SxuRegtPMbPHzWy1mf3WzHbNijUOuCNUf/vDvwC/DyHcF6K/p1/fv+1mVUwCbiO++aLqQSyE8CwwGzigmGZmfVOdjCuMn2Fm09P395rZfalXdpmZXW9mQystL9XXIdnwWDNbX1jmlNRDu9LM7jWzqm/uCSE8DbxIfDVfJeOA2wtl+YudANQAAAvrSURBVIqZPZm220IzO8fM+qS0H5rZzML0Y9O0O6Thfc1sTlrv0vz9UlqpbXzazB4H1gJDzezjZvZQ6n1+wcwuL+WX5nuTmc1KbXVumj+Y2YhsmtNTb/gqM3vAzI6utNJl6neGmV1jZj9N9bs47R/vMLP/S+t3l5m9OZunw8y+Y2b3pP3Am9lBWXrVNmBm/dI2fSrlP8/MTrDYgz8FGGsbe9xHVViPw9MyVqVtNilLG2tm683sxJT3KjO7Id+Py+TXnWPFaDP7XVrPZ9P8fbL0d6a6WWNm9xB/9OXLHGBmF5jZfDNbbma3mdmelcpYpsxvMLOfWTxWLTGzqy27MmGFqx1ZGxxWqa7N7LS0vt9M7XGpmV1Yph0Py/I9zcyeSd//CzgUOCvl+VSFsk81szvN7LzURl4ys6+b2R6pTleb2V/M7O3ZPJu1r2RtfVrW1jdpN+l71foprEunWx8atN1vJx6jNl8IQR999GnyB+ggvlbs7UA/4FpgHnAFsAMwnPhqvJPT9NsBzxAvtW4P7ALcCvw0y3MCscfRgPcAfweOSWkjgEAM2AYDg4B7gWnZ/PcBXy6UcyzQkQ2PB/4BnE18P/TOFdZtQlfjgSHAK8DxxCAyAGMKy16fDe8JPJWvcyH/84GZ2fBAYA1waBo+BDgI6Au8Cfg9cF02/Qzgymw4AIdUKc/3U52NAvoQe31fBHbJ67xMOWcBZ1dpG38FPlQY91FgZNq2B6RpJqW0fYB1wJBs+quBq9L3ocBLxIC+P7Ab4IHvFNrGnale+qf1eR/wz8SOiD2Jr2k8J1vGncD/pLY0FLg75TMipZ9ObLP7pzzen7bHnhXWu1i/M4ht+Lg0/+Q0/6+Jr+0bAPyOzm24A3geGJPW41vAMmBQjW3gvLSeo1NdDwNGp7SpxB9k1fbrkanMp6VlHAwsB8Zn6xiIr7kcCLyReBz4dgOPFTul9nEWsG2a71ngjCz9pVQ3/VN9LKHzfv5z4rHijWma/wCeBPqV21fKlPk2YjvfJX1uAW6pciwYkeplWKW6TnX6KnAJ8Rj4FuIrLqeUyyOb55ls+G7gzC624dS0nM+wcT94DbijsA1uz+bZ3H1lBrHdfCjlcXwqwx4V9o1K9fNMYdyG7dSI7Z6mGUO8ItW/Wj3W8mnpSVcffbbWTzrgnpENvz8dgPKg4QbgovT9BGBeIY8xxICtT4Vl3AScn76XDsYHZelfAB7IhucCpxXyGJsfkNK4DwD/SzypvUa8BL9vYd1eBlYWPq/T+STzr8QTZekkdj9weWHZIc27ApgPXEaZIDdN/3Zi4DU0DU8E5lbZBh8AlmbDGw7OabhiwEkMRlYDhxXyfKS0jlQOOH8OXFqlXOuAsV20nwuAG7Lh+4Cvpe87pvp/dxr+BvC7wvwfJZ2csrZxWBfL/CLw5/R9WJpnVJb+XjqfRB8FTi3kMYsKJ3zKB5x5kDIg5T8+G/d5OrfhDuB72bABC4FPdNUG0rRrgOMqTDuVrgPOKcC9hXHnAHMKbTrfz38I3Fwlzw7qO1Z8AniO9KrqNG4S8FT6fnKqkzz9+6T9nPiDNADDs/RtgFWk/YEqASfxR28A9srGvTWN2zVbp+4EnK8AA7JxnyHt48U8snm6E3A+Vhi3tMw2WNHAfWUGWVtP45YBH66wb1Sqn2oB52Zv9zRurzTd0Gr1WMtnQ5e8iDTdC9n3tcT7FZcVxpUutY0EhtumTyoGYk/NYjP7MrFXaRjx5Lk98SGVSst8OcsfYlBX7d7CuMAQZhN/BWNmbwMuBWab2ciQjkjE3rdr8/ksexrSzCyV9doQwqtp9FXAuWb2jRDC6jTutVDjgyQhhCfM7H5iT++PgE8B07NljgF+QOxxG0Cso4FlsqrF4DTvLMueRCf2fgwrP8sGg4jBcyWbbAeL985+ndib2pfY+/CnbJLpwOeID319DFgUQrg3pY0E3l1oO0bsvcl1FJZ5FPAd4G3EnrI+xBMvxF5SiCewkgWF/EYCl5jZxdm4vsAiarehvYYQ1sZms8l+U7wc3ZHNE8xsIWmbdNEGhhB7DOfWUb6i3dl0284DPpwNF/fz4n5YTj3Hit2BBdm+WCrD7un7sDLpeZlHpr8Pp/ou6ZflUU1pmjzPeVnaC3Tf0hDC2my4g673t+4olnEtVdpdA/aVcsuspV3Uo1HbfRAbOwI2i+7hFOmdFhB/ye9c+GwXQlhsZu8mXg6cBAxOQdos4gm1Vg8QL8/WLITwJDHI2YN46axW7yFeeppYus+LePlmILGHprumA6el+44OBn6WpV1P7EXdO4QwiPIPKeXWEAOQkjdn318knhCOLGyPHUII53aR777Euq6k03Yws92Jl/DOJvYQ7US8rJhv2+uBvc3sQGJPx/QsbQGxNyQv504hPoiVez1bZn9gZsp3eKqvb2bLXJz+Ds/mz7+XljuxsNyBIYTPVVn3RhhR+pJ+2AxnY5BbrQ0sIwYSe1XI9/UK43PP5ctPRqXxrfIcsId1jhryMiwukz4i+14KhvYqbLsBIYTralx+Mc9RhbTVVN63oHJdDzWzAYVyl7Zt6Udqd/LttgbtK/Uqtx7FOoXO69+o7b4vsQd4XTfLvoECTpHeaTbQ3+IDDTtatJuZfSSlDyJe3l4GBDM7jnhfUT1mEi/1VGRmE81svKX/JZlu0J8MPB5CWF7HsiYR7597G/CO9NmXGCh9ts5y564nBrIXE++xWpylDSJeHlptZsOJ9zJV8xfgk2bWP93c//VSQuol+DFwgZntBWBmAy3+H9PiSW6DFAgPId4PVslMOj9UNJB4bF4GvGpmBwOn5DOEEFYCNxOD0oOJ93CW/AxwadttZ2bbpIcMjq1Shv7EnpoVIYS/m9k+xMuEpeUtIl6ePDe1xyFA8d/NXARMtfiQj5nZ9mZ2SOoVb6aJZnagxYdJziD2ZN6S0iq2gbRNLwXOt/iQlVl8iGV0mmQJ8SpD/yrLvg4YY2anWnyo7J3Etn5VQ9ewuluI225KartvJQZApTLMJrapMyw+JHUg8f5jAEIIS4lXRi619O9vzGxnM/uIFf51WTkhhOeB3wIXpvl2AS4EfhNCKPXi/QU4Ke0zQ4j3m+Yq1fU2wHmpLY0i3i5ydVruS6QfORb/08J+xKsoxXxrfvipRo3YV+pVrn4eJAbkH0j7+EeAw7L0Rm33o4jHqM2mgFOkF0qXkd5D7Pl6knjSvJMYqAHMIQYWfyb2vp1ADEDqMQdYb2Zjq0yzgnjp9gkze5l47+BK4r1wNbH4VPA44IIQwpL8Q+ylPcC6eNq7khDCKuJ6v4/4L4hynyXe87WaeA/qjV1k90XiyWk58R65GYX0fwd+BfzKzP5GfLBjMtWPoxOBGamclVwD7J9OqIQQnsiWtZIYJJXraZpOXO852YmdVK9HEOu8g7gNb6bwhGouhLCGuJ3PN7M1xB7V4u0ZnyAGc4uID6CV6vOVlMc04oNc09MyFxIDi35V1r0RriD+4FgBnEi8J7NU3121gW8Tt/XMNM3dbAxQbiT20C2x+CTxyMK8hBDmE+/v+yLxAY1rgLNCCDc0auW6ktb1aOKPlr+y8djwo5S+kvgg1onEOroY+O9CNqcTH9C728xWE+9NHk+8lFqLCcT6e4p4vFoJnJqln0n8gfwCsY6vL8xfqa4XENvbfOKx5zZiGyv5JPFYtCqtbzHQv4j442ulmT1W47pU1Yh9pRs2qZ8Q/43aV4jtfzlwLPFBpVI5N3u7m9nOxPZ9WTfL3Yl1vrwvIluT1Os1JYRwWBoeSwyQRvRkudpR6hWdH0KwNDyE+HS4K9x/V27eycSHfk6pNl1vYmbHEIPi7UMPnUgs3id8ZvH+YWl/ZnYacds2uoey5XrDvtIdZnYO8f7hhvzzfD00JLIVCyHcRuw1kAZLQeYeNU57GQ3qRWgWM3sH8V6yR4gPHJwN/LKdTqAirbCl7CshhH9rZH66pC4iuQ7a+80+PWkl8UGoLdUuxMvSa4B7gIeJl/REpDPtK2XokrqIiIiINJV6OEVERESkqRRwioiIiEhTKeAUERERkaZSwCkiIiIiTaWAU0RERESa6v8B+6E/9UfFKXcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 576x396 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "shap.summary_plot(shap_values_mgbm, X, plot_type=\"bar\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/navdeep/article-information-2019/models/MGBM_BEST_GRID_v2_HMDA_1575927005.5342216/mgbm_grid2_model_28\n"
     ]
    }
   ],
   "source": [
    "# Save the model\n",
    "model_path = h2o.save_model(model=best_mgbm2, path=\"../models/MGBM_BEST_GRID_v2_HMDA_\" + str(time.time()), \n",
    "                            force=True)\n",
    "\n",
    "print(model_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# GBM Grid Search (Unconstrained)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gbm Grid Build progress: |████████████████████████████████████████████████| 100%\n"
     ]
    }
   ],
   "source": [
    "# Train and validate a random grid of GBMs\n",
    "gbm_grid2 = H2OGridSearch(model=H2OGradientBoostingEstimator(distribution='bernoulli',\n",
    "                    # more trees is better if the learning rate is small enough \n",
    "                    # here, use \"more than enough\" trees - we have early stopping\n",
    "                    ntrees=10000,\n",
    "                    # learning rate annealing: learning_rate shrinks by 1% after every tree \n",
    "                    # (use 1.00 to disable, but then lower the learning_rate)\n",
    "                    learn_rate_annealing = 0.99,\n",
    "                    # score every 10 trees to make early stopping reproducible \n",
    "                    # (it depends on the scoring interval)\n",
    "                    score_tree_interval = 10,\n",
    "                    # fix a random number generator seed for reproducibility\n",
    "                    seed = 1234,\n",
    "                    # early stopping once the validation AUC doesn't improve by at least 0.01% for \n",
    "                    # 5 consecutive scoring events\n",
    "                    stopping_rounds = 5,\n",
    "                    stopping_metric = \"AUC\",\n",
    "                    stopping_tolerance = 1e-4),\n",
    "                    grid_id='gbm_grid2',\n",
    "                    hyper_params=hyper_params,\n",
    "                    search_criteria=search_criteria)\n",
    "gbm_grid2.train(x=X, y=Y,\n",
    "                training_frame=train,\n",
    "                fold_column=fold_column,\n",
    "                ntrees=100,\n",
    "                seed=SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get the grid results, sorted by validation AUC\n",
    "gbm_gridperf2 = gbm_grid2.get_grid(sort_by='auc', decreasing=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "          col_sample_rate learn_rate max_depth         sample_rate  \\\n",
      "0                     0.5        0.1         8  0.7000000000000001   \n",
      "1      0.7000000000000001       0.08         9  0.6000000000000001   \n",
      "2                     0.9       0.08         6                 0.8   \n",
      "3                     0.4       0.04         9                 0.5   \n",
      "4                     0.8       0.05        10  0.6000000000000001   \n",
      "5      0.7000000000000001       0.04         8                 1.0   \n",
      "6      0.6000000000000001       0.04         7                 0.8   \n",
      "7     0.30000000000000004       0.04        10                 0.9   \n",
      "8                     0.9       0.04         7                 1.0   \n",
      "9                     0.9       0.06         6                 0.9   \n",
      "10                    1.0       0.09         5                 0.5   \n",
      "11                    0.8       0.09        10  0.7000000000000001   \n",
      "12                    1.0       0.09        10                 0.9   \n",
      "13     0.7000000000000001       0.09        10                 0.5   \n",
      "14    0.30000000000000004        0.1         6  0.7000000000000001   \n",
      "15                    0.4       0.08         5                 0.8   \n",
      "16    0.30000000000000004       0.08         6                 0.8   \n",
      "17                    0.4       0.07         5                 0.5   \n",
      "18                    0.2       0.09         6                 1.0   \n",
      "19                    0.2       0.06         7                 0.8   \n",
      "20     0.7000000000000001        0.1         3  0.7000000000000001   \n",
      "21     0.6000000000000001       0.05         4                 0.9   \n",
      "22                    0.9       0.04         4                 0.5   \n",
      "23                    0.4       0.01         6  0.6000000000000001   \n",
      "24                    0.2       0.03         7                 1.0   \n",
      "25                    0.9       0.06         3                 0.8   \n",
      "26                    0.2        0.1         3                 0.8   \n",
      "27                    0.2       0.06         4                 0.9   \n",
      "28                    0.1        0.1        10                 0.5   \n",
      "29     0.7000000000000001       0.08         2                 0.5   \n",
      "30                    1.0       0.07         2  0.6000000000000001   \n",
      "31    0.30000000000000004       0.06         2  0.7000000000000001   \n",
      "32                    0.1       0.01        10                 0.5   \n",
      "33                    0.1       0.09         3                 0.9   \n",
      "34                    0.1       0.02         6                 0.5   \n",
      "35                    0.1       0.01         6  0.6000000000000001   \n",
      "\n",
      "             model_ids                 auc  \n",
      "0   gbm_grid2_model_20  0.8238587168591304  \n",
      "1   gbm_grid2_model_27  0.8232568997340003  \n",
      "2   gbm_grid2_model_11   0.823206306486943  \n",
      "3   gbm_grid2_model_19  0.8231330212240818  \n",
      "4    gbm_grid2_model_4  0.8229603810197769  \n",
      "5   gbm_grid2_model_34  0.8229469663325818  \n",
      "6   gbm_grid2_model_33  0.8225198438411869  \n",
      "7    gbm_grid2_model_2  0.8221293573015995  \n",
      "8   gbm_grid2_model_35  0.8221011062807465  \n",
      "9    gbm_grid2_model_7  0.8220914357449876  \n",
      "10  gbm_grid2_model_24  0.8217533345953516  \n",
      "11  gbm_grid2_model_26  0.8215544160855675  \n",
      "12  gbm_grid2_model_28   0.821486002494532  \n",
      "13  gbm_grid2_model_36  0.8213460198584795  \n",
      "14  gbm_grid2_model_12  0.8202215415918771  \n",
      "15   gbm_grid2_model_3  0.8198484454511903  \n",
      "16  gbm_grid2_model_16  0.8190340787092906  \n",
      "17   gbm_grid2_model_6  0.8189682688353633  \n",
      "18  gbm_grid2_model_22    0.81655681109785  \n",
      "19  gbm_grid2_model_31  0.8160245310763712  \n",
      "20  gbm_grid2_model_18  0.8156559614936468  \n",
      "21  gbm_grid2_model_30  0.8150778265271272  \n",
      "22  gbm_grid2_model_13  0.8136168348573143  \n",
      "23   gbm_grid2_model_1  0.8115745253805452  \n",
      "24  gbm_grid2_model_29  0.8108750632975006  \n",
      "25  gbm_grid2_model_17  0.8107895060083133  \n",
      "26  gbm_grid2_model_25  0.8083008438503759  \n",
      "27  gbm_grid2_model_14  0.8074911829029933  \n",
      "28   gbm_grid2_model_9  0.8073657996966307  \n",
      "29  gbm_grid2_model_15  0.8032499501650014  \n",
      "30   gbm_grid2_model_5  0.8016645035337913  \n",
      "31   gbm_grid2_model_8  0.7943929740267996  \n",
      "32  gbm_grid2_model_10  0.7938513823879797  \n",
      "33  gbm_grid2_model_21  0.7938138299918746  \n",
      "34  gbm_grid2_model_32  0.7925067145742896  \n",
      "35  gbm_grid2_model_23  0.7887380137418685  \n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(gbm_gridperf2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8277469182063814"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Grab the top GBM model, chosen by validation AUC\n",
    "best_gbm2 = gbm_gridperf2.models[0]\n",
    "\n",
    "# Now let's evaluate the model performance on a test set\n",
    "# so we get an honest estimate of top model performance\n",
    "best_gbm_perf2 = best_gbm2.model_performance(test)\n",
    "\n",
    "best_gbm_perf2.auc()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Shapley Importance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "First 5 rows of contributions:\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead>\n",
       "<tr><th style=\"text-align: right;\">  term_360</th><th style=\"text-align: right;\">  conforming</th><th style=\"text-align: right;\">  debt_to_income_ratio_missing</th><th style=\"text-align: right;\">  loan_amount_std</th><th style=\"text-align: right;\">  loan_to_value_ratio_std</th><th style=\"text-align: right;\">  no_intro_rate_period_std</th><th style=\"text-align: right;\">  intro_rate_period_std</th><th style=\"text-align: right;\">  property_value_std</th><th style=\"text-align: right;\">  income_std</th><th style=\"text-align: right;\">  debt_to_income_ratio_std</th><th style=\"text-align: right;\">  BiasTerm</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td style=\"text-align: right;\"> 0.0158299</td><td style=\"text-align: right;\">  0.00414181</td><td style=\"text-align: right;\">                   7.82996e-05</td><td style=\"text-align: right;\">         0.171057</td><td style=\"text-align: right;\">                 0.901487</td><td style=\"text-align: right;\">               -0.00635005</td><td style=\"text-align: right;\">           -0.000623261</td><td style=\"text-align: right;\">            0.328048</td><td style=\"text-align: right;\">  -0.168341 </td><td style=\"text-align: right;\">                 0.216464 </td><td style=\"text-align: right;\">  -2.83641</td></tr>\n",
       "<tr><td style=\"text-align: right;\"> 0.0273239</td><td style=\"text-align: right;\">  0.00585237</td><td style=\"text-align: right;\">                  -0.00100548 </td><td style=\"text-align: right;\">        -0.103742</td><td style=\"text-align: right;\">                 0.443703</td><td style=\"text-align: right;\">               -0.0167528 </td><td style=\"text-align: right;\">           -0.000612409</td><td style=\"text-align: right;\">           -0.304953</td><td style=\"text-align: right;\">   0.0121991</td><td style=\"text-align: right;\">                -0.420809 </td><td style=\"text-align: right;\">  -2.83641</td></tr>\n",
       "<tr><td style=\"text-align: right;\"> 0.0053243</td><td style=\"text-align: right;\">  0.0122485 </td><td style=\"text-align: right;\">                   0.000636898</td><td style=\"text-align: right;\">         0.246282</td><td style=\"text-align: right;\">                 0.52934 </td><td style=\"text-align: right;\">               -0.483805  </td><td style=\"text-align: right;\">           -0.336803   </td><td style=\"text-align: right;\">            0.44275 </td><td style=\"text-align: right;\">  -0.0115226</td><td style=\"text-align: right;\">                 0.0393331</td><td style=\"text-align: right;\">  -2.83641</td></tr>\n",
       "<tr><td style=\"text-align: right;\"> 0.021757 </td><td style=\"text-align: right;\">  0.00565331</td><td style=\"text-align: right;\">                  -0.00105577 </td><td style=\"text-align: right;\">        -0.161554</td><td style=\"text-align: right;\">                -1.23905 </td><td style=\"text-align: right;\">               -0.0860368 </td><td style=\"text-align: right;\">           -0.0493688  </td><td style=\"text-align: right;\">           -0.541179</td><td style=\"text-align: right;\">   0.0182554</td><td style=\"text-align: right;\">                -0.245621 </td><td style=\"text-align: right;\">  -2.83641</td></tr>\n",
       "<tr><td style=\"text-align: right;\"> 0.0121323</td><td style=\"text-align: right;\">  0.00453242</td><td style=\"text-align: right;\">                  -0.000120826</td><td style=\"text-align: right;\">         0.196161</td><td style=\"text-align: right;\">                 0.939752</td><td style=\"text-align: right;\">                0.00398349</td><td style=\"text-align: right;\">            0.00293035 </td><td style=\"text-align: right;\">            0.438099</td><td style=\"text-align: right;\">  -0.133914 </td><td style=\"text-align: right;\">                 0.219434 </td><td style=\"text-align: right;\">  -2.83641</td></tr>\n",
       "</tbody>\n",
       "</table>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "shap_values_gbm, expected_value_gbm = get_shapley(best_gbm2, train, X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAApwAAAFeCAYAAAAog8+HAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xm8X9O9//HXiaFKEmNQRAaUnw6qfbdoayhu7iVOtdSNMULjohR1ETU1QqURGjW0YqiQmIeWIKZWlGvqB6VVqSITTQiSVBJtJDm/P9b6xvZ1hm+Os3NOkvfz8TiP8/3utfdaa+/vCe+z1tr71DU0NGBmZmZmVpZO7d0BMzMzM1u+OXCamZmZWakcOM3MzMysVA6cZmZmZlYqB04zMzMzK5UDp5mZmZmVyoHTzMzMzErlwGlmZmZmpXLgNDMzM7NSrdzeHTBbnowdO7ahvr6+vbthZma2tNTVspNHOM3MzMysVA6cZmZmZlYqB04zMzMzK5UDp5mZmZmVyoHTzMzMzErlwGlmZmZmpXLgNDMzM7NSOXCamZmZWakcOM3MzMysVA6cZmZmZlYqB04zMzMzK5UDp5mZmZmVyoHTzMzMzErlwGlmZmZmpXLgNDMzM7NSOXCamZmZWanqGhoa2rsPZsuNugsW+B+UmZl1SA0nrVxGtXW17OQRTjMzMzMrlQOnmZmZmZXKgdPMzMzMSuXAaWZmZmalcuA0MzMzs1I5cJqZmZlZqZbLwClpvKQz2rsfHYmkBknfbO9+LAlJL0rq1979AJC0QNIu7d0PMzOzZVEpD2QykDQAOCMiNm/vvnR0knoCE4HuEfF6ZXtEfG4ptD2J9DmNKbstMzOzFdVyOcJpHYekVdq7D2ZmZta+lvsRTklfBC4CtgVmAr8GhkbEwlx+DbA7sBYwFTg3Im7IZbsADwEHAecB6wH3A9+PiPeaaXMH4HJgVUlz8ua9ImK8pJ2B84GtgGnAiIgY2cI59AWuATaOiA/yts7AdKBvRDwi6Txgf2B94E3gkoi4qIn6BlA1+ippFLAgIgbm95sCPwe+CTQAY4H/be6883GDgZ2AZ4FD8vc9mrvOwPP5+98kNQDDIuKc6tHH1ly7fNz+wE+ATYB5wH0RcaikscCmwFWSLgcej4g+kroAlwL1wHvAWS21YWZmZk1brkc4Ja0JPAg8DGwI9AUOB04s7PYY8CVSEBoCjJK0daF8JaAPsA3wWVJwPa65diPiCeAo4LWI6Jy/xkvqBdwH/ApYFxgADJW0Xwunch+wIPe/Yj9S4PxDfv9XUjjsAhyR6/3PFuptlKTVgN/nOnsBW5PC2i9qrGInUiDsDuybtzV3nbfJ37fM1+qcRvrUqmsnaXVgNHBMRHQBegNXAUREPTAFGJjb7ZMPuwjYIp/3F4G9ST8HZmZm1grL+whnX2A+aTStAXhJ0jBS4BwOEBFXF/a/SdJJwC6ksFVxakTMAeZI+i2gVvbnAODZiBiV3z8paSQwELi1qYMiYqGk0cBhwG/z5sOAa/J5UbUG8feS7gF2I43ILqm9gLqIqIzsvS/pTOBxSUdURoebMTkiLsyv5+f+1XKdm9Oqa5d9AGwl6U8R8S7waFM7SupEGtHuGxHT87ZBwHdr7KeZmZlVWd4DZ3dS+GkobHs1b6+Ei8FAP9IIaAOwBtCtsP/CiJhReD+XNIrY2v5MrNr2KmkErSXXAC9IWj+3/3XgwEqhpONII5ubAHXAp4EbGqmnFr2ATSXNqtreQLpOb7Rw/OTimxqvc0tade0iYp6kPUm/ZPxU0mvAhYXp/GrdgE8Bkwrbqts1MzOzJbBcT6mT1gr2kFRX2NY7b4c0ajaQNO27dkSsRVpPWMcnt6iJ/vSs2lbsT5MiYgLwDHAwaTr5ocod3ZK+AQwDjgTWy+cxlqbP4z1S4CvaqPB6MvByRKxV9bVaRLQUNuHj597SdW7sWlX7JNdufER8m7QG91xgjKTNmmj7bdKobLGt6nbNzMxsCSzvI5z3kNbjnSZpOGnkbhBQudGkK2lt5AygU76ZZhvg7jZoezqwvqSuEfHPvO1G4ExJ/Umjj18mhcSja6zzGtL60S7AyYXtXYGFpPNoyDcZ7UHTU81/yn3bC7iXNEq4E1CZlr+bNBp4GnAJMIcUSL8WEb+psa9FLV3nGaTgtwXwemMV0MprJ2kD0trWhyJidmHUtrIsYHpuF1i8fOEG4GxJfwHeB35W+6mamZlZteV6hDMiZpNu+NmddOf2/cB1pLuvAa4FngJeIU0Tb00z6/uW0MOkG5YmSpolaeeImAjsCRwLvEO6meXMiLilxjpvIo3qdQbuLGyvnNfTpBG67wFNBsOIeBU4HrgCeBf4L+D2Qvk8YFfS9ZgAzAZ+R7rppzWavc4R8T5wJnBjvlanN9Ln1l67TsAxwCRJ7wGXAYdGxKRcfi5wsKSZksblbceTptEnAH8mjRa3tG7VzMzMmlDX0NDQ8l5mVpO6Cxb4H5SZmXVIDSeVMrFd0zLE5XqE08zMzMza3/K+hrM0knYExjVRfF5EnLeE9Z0GnNZE8R4R0VZT/Z9IW593W1hWrp2ZmdmKylPqZm1o7NixDfX19e3dDTMzs6XFU+pmZmZm1v4cOM3MzMysVA6cZmZmZlYqB04zMzMzK5UDp5mZmZmVyoHTzMzMzErlwGlmZmZmpfJzOM3akP+0pa0oSvoTeWa27PFzOM3MzMys/TlwmpmZmVmpHDjNzMzMrFQOnGZmZmZWKgdOaxeSdpG0oL37UStJCyTt0t79MDMzWxY5cK5AJA2Q9Ep796MjkTRJ0sHt3Q8zM7PlmQPnUiZplRWpXTMzMzM/SK0NSJoE/BroA3wJmAAcHRF/lDQKWAX4APg2cDNwtKSdgfOBrYBpwIiIGJnr2wV4CPg+MAToAtwFHBsRc/I+6+bj+wCrAQ8DP4yIN6v69C3gq8BPgZ8Aq0qak7u+FzAMuCkiRhTOZwjwjYjYrZlzHg5sERHfKWzbBRgLbAg0AGOArwOrA68AgyLiwSbqGwUsiIiBVdf1jIgYk9/vCAwFtgZmAr8Efh4RzT77UtL++dw3AeYB90XEoZLGApsCV0m6HHg8IvpI6gJcCtQD7wFnNVe/mZmZNc8jnG3nKOB4YB3gNuBeSV1z2X7AOKAb8L+SegH3Ab8C1gUGAEMl7VeobyVS4Pki8P+AzwI/B5BUB/yWFOo+D/QgBaMbqvp0BHAiKbBelPv4WkR0zl/jgZGkYEuuuxNwGHBlC+d7DbCnpG6FbYcBt0TEXNLP1h3AFvkcbwRur9q/ZpK2Bu4FhpOuY1/gWOCQFo5bHRgNHBMRXYDewFUAEVEPTAEG5uvRJx92Ue731qTrvzfp8zAzM7NW8Ahn27k6Ip4BkDQM+AFpBBHgsYi4Ob+eJ+kA4NmIGJW3PSlpJDAQuLVQ56CImA3MlnQWcLeko4AvA18Bdo+If+c2TwHelrRJRLyej78yIp7Lr9+X1Fi/bwJGSNo+Ip4E/hP4NPCb5k42Iv4q6Tng4Hx8F+B7pBFX8kjsmMIhwyUNIo223ttc3U34AXBrRNyZ30+QdCnQH7iuhWM/ALaS9KeIeBd4tKkdc+A+COgbEdPztkHAd1vRZzMzM8OBsy1NqryIiAZJU0hTuB8py7oDE6u2vUoaSSuaXFX/p4D1gF759ZtVIfJfpCni1wvHNCsi5kkaQwq7T+bv11WCbAuuAY4GRgD/DbweEf8HIOnTpNHIPXOfF5FGWls1wkk6510l7VPY1gmY2txB+fz2JI30/lTSa8CFEVE9GlzRjXRtJxW2VX9WZmZmtgQcONtOz8qLPOVdCX5bk8JW0VRSECvqzcfDUw9SEK3U/2/gbVIQnQusExHVdRdVlzW170jg/yT9lDSN/6Vm6iyqjI5+mbQs4JpC2YnATsBuwKQcwt+m6b+5+h4pmAIgaWVg/UL5ZODXEXFMjX1bLC8dGC9pJdI62tslPRURr/Lxa/I2MJ90vYvX3szMzFrJazjbzuGSvpzvBj+ZdKPMPU3seyPwFUn9Ja0s6WvAkcDVVfsNldRV0vrAYGB0DpgBPA9cnG8eQlK3fHNMc6YD6xfWlgIQES8ALwK3A09HxF9rOeGImEWaej8X2B64tlDclRSQ3yHdqHQWsFYz1T0D7Capl6RPkW5yKt5Z/0tgf0n1klbJ123rfPNVkyRtIGlfSWtGxEJgVi5amL9PJ63XrJzTQtJa2LPzsV2BnzXXhpmZmTXPgbPtXAFcTLp7uh9pDeDsxnaMiImkEc5jSYFsNHBmRNxS2G0hKbD+Gfgb8Bpp1JAcOvcmjRY+I+k90nT4Li308WHgQWCipFlVYW0ksC0t3yxU7RpgD+D+iJhW2P5zUrj7B2mkcB7NT/FfT7oT/9m8/xTgjUphRPyFtCb2BNJd/W8Bo2h5ir4TcAwwKV+ny4BDI6LSl3OBgyXNlDQubzueNI0+gXT9x/JhQDUzM7MlVNfQ0OwTZawG1Y/vaYP6dgEeioiltuQht/lbYKOImLe02l3e1F2wwP+gbIXQcJJXZJkZ0PRSuY/wCKchaTXgJNJd7Q6bZmZm1qb8K+oKLt/1PYY0lf3TqrJxwI6NHRcRncvvXW0knQac1kTxHhHR5GOQzMzMrHyeUjdrQ55StxWFp9TNLKtpSt3/xTBrQ3dtOY76+vr27oaZmVmH4jWcZmZmZlYqB04zMzMzK5UDp5mZmZmVyoHTzMzMzErlwGlmZmZmpXLgNDMzM7NSOXCamZmZWan84HezNuQHv1uRH45uZisA/y11MzMzM2t/DpxmZmZmVioHTjMzMzMrlQOnmZmZmZXKgdPMzMzMSrVMBU5Jm0hqkNSzhn0HS3poKXSrVSRdLunS9u5HR9aRrpGkhyQNbu9+mJmZLYtW2Gd25NA6EegeEa/XsP8A4IyI2Lwt2o+Io9qinuWFpEmk6zumsm1pXCNJo4AFETGw7LbMzMxWVMvUCKcteySt0t59MDMzs/bVoUc4JW0IXAHsDLwJnF9VfgRwPNAdeA0YFBEPFHapkzQC6A+8D1waET/LZc/n73+T1AAMi4hzmujHDsDlwKqS5uTNe0XEeEk7535tBUwDRkTEyBrObRSFkbXch2OAw3JdLwIDImJCLl8FOBk4FNgIeCuf722SVgZOAwYAawPPAsdHxF8Kba0EfADsA8wFTgJeAq7M7QVwUET8Ix+zOjAE2BdYE3gaODYiXmnhvHYBHsrncTbQDegi6XjgaGBjYCZwPWlEc6GkscCmwFWSLgcej4g+jVyjHsDFwDdIn+ftwI8j4v0W+rQtcAnwBWAhMAHoCxwBHJT32T/vviawCDiV9HmsDlxLjQ+2NTMzs4/r6COc15MCwqbATqRABSwOm4NIgWFt4HTgDknFKe+dSEH1M8DewImSDsxl2+TvW0ZE56bCJkBEPAEcBbyW9+2cw2Yv4D7gV8C6uX9DJe3XyvMdQAp46wFTSSGp4lzgYGA/oCsphL+cy04mheo9gQ2BR4EHJXUtHP89UkBbBziHFDSHAN8FNgAaSAGxohJEt891PgXcXeOI5Uq5L9vmugFeB/bIfd8bOBwYCBAR9cAUYGC+tn2qK8yh+h5gOtAj9+sbwAU19Ocy4IF87hsAJwLzI+J80s/YtYXPdSHpOv8o93ND4G3Sz5KZmZm1Qocd4ZS0MbArsHlEzAZmSzqbFBwgjWwOiYjKSOW9kh4G9ieFM0gjjsMiogF4RtIVpFB3Qxt18wDg2YgYld8/KWkkKUjd2or6hkfEFFg8Kjkmv64jjbb1i4gX8r6v5y9Io4nDCqOhQ3If+gI35n1+HxH35PLrSCF5dGX9qqTbSCN+SFoPOBDoERFv5m1nAycA2wGP1XAug/LnBkBE3F4oe07SaGA3oMXR4OxrwBbAdhExF5gr6Qzgt5KOzZ9xU+aTfmnpHhGTgCdbaKs/MDIingGQNJT0C4eZmZm1QocNnMAm+fvkwraJhde9gMskXVzYtjIfhjCAyVVBZBJpSrmtdK/qE8CrpJGx1phWeD0X6JJfdwPW4MMRzWb7ERGL8k043RurOyLmSapub16hvV75+wt5v4pVqupsyiLSCO1ikg4gjSz2Jn1Oq9Jy8CvqDszIYbPiVWA10vV5q5ljDwPOBB6T9AEpyJ8dEQua2H8T0s8KsPh6Tm5iXzMzM2tBRw6cb+TvPUjBAqBnoXwy8JOIaG4ksYekukLo7MmHgXTREvansf2nkqaOi3pTFbbawAxSINwC+HsT/ehZeSOpU37f2n5UwtUWETGjFcc3FIO+pO6kkLcPMC4i5ku6ACim2ZY+j6lAN0mrR8S8vK038C/S9WlSREwkTeEj6QukUfKJwK+baPcNPno960g/h2ZmZtYKHTZwRsTrksYD50s6DPg0cFZhlxHAYEl/J90AtBrwFeDtytQyae3myfnGoc+TpoxPzGUzSGFjCz46KtqU6cD6krpGxD/zthuBMyX1J03Tfxk4knRzTJuJiAZJvyRdiymkG4o2BtbJU+yjgFMk/YE0MjeI9Nne08r23pJ0A/BLSSdExBuS1gK+BTwYEXNaqKJaZ9J64RnAB5K2Bw4h3bRUMZ30WTTlaeAV4EJJ/wusRVqLek0L0+lIOjT3+x/ALGABaW1wpd3tJXWKiEr4HE261r8B/ky6wWrDWk/WzMzMPqqj3zR0IPAp0ujWo8B1lYKIuJJ0d/g1pLuep5CmTYs3tTxKCp3TgbuBX5DXb+Y7m88EbpQ0S9LpLfTlYeBBYGLef+c8crYncCzwDimonBkRt3ySk27C6cAtwG+B94DxQOUGqeGk8PsA6SapXYE+hWDcGkcAfwPGS3qPFLz2I91ctEQi4iXgJ8CdpMB3Kh+uLa04FzhY0kxJ4xqpYwGwF2m6ewopgD5FCoMt2ZW0hncu8ATpZ2B0LruKtFzhnfy5rkT6ObsEGEu6nusDf6j5hM3MzOwj6hoaljg/mFkT6i5Y4H9QtljDSR12EsnMrK3U9NjAjj7CaWZmZmbLOP/6nUnaEfjYVG52XkSct4T1nUZ6GHtj9oiIR5ekvo6k8PD7ao9GxB5LtTO0/WdnZmZmbctT6mZtaOzYsQ319fXt3Q0zM7OlxVPqZmZmZtb+HDjNzMzMrFQOnGZmZmZWKgdOMzMzMyuVA6eZmZmZlcqB08zMzMxK5cBpZmZmZqXyczjN2pD/tGX785+TNDNbqvwcTjMzMzNrfw6cZmZmZlYqB04zMzMzK5UDp5mZmZmVyoHTzMzMzErlwNmOJI2XdEZ798NaJukVSQPaux9mZmbLIgdOW+ZI2kXSgjaqy6HfzMysZA6cZmZmZlYqPyG5g5D0ReAiYFtgJvBrYGhELMzl1wC7A2sBU4FzI+KGXLYL8BBwEHAesB5wP/D9iHivhXa3AS4GPgesBDwJHBsRr+byUXn7B8A+wFzgJOAl4EpgKyCAgyLiH/mYdYERQJ/czP3AjyLi3Vw+CTgjIsbk9z2BiUD3iHi90Oa/gP1ym0MiYqSkjYBxwEqS5uT6j4mIa5s5x92B4cBmwHzgTxGxu6RLgR2BHSSdCrwREVtKWgUYBhwMLMrnYmZmZq3kEc4OQNKawIPAw8CGQF/gcODEwm6PAV8iBc4hwChJWxfKVyIFvG2Az5KC63E1NN8ADAY2BnoCc4AxVft8D7gdWAc4hxQ0hwDfBTbIdZxd2P96YG3g/+Wv9YDRNfSlus2xuc0fApdK6pFD7R7AwojonL+aDJvZdaRQvWY+z3MBIuJY4FHgnFzPlnn/U4G9gK8DvUjXpccS9t/MzMwyj3B2DH1JI2/nRkQD8JKkYaTAORwgIq4u7H+TpJOAXYC/FrafGhFzgDmSfguopYYj4oXC239LOhv4s6TVI2Je3v77iLgHQNJ1wK+A0RHxet52G3BEfr0R8J/AZyNiZt52IjBB0mciYlqN1+T3EXFXfn2HpFmkwD25xuOL5pNGNzeIiOnA+Bb27w/8LCJeAcjX+vutaNfMzMxw4OwougOTc9iseDVvR1In0ihkP9IIaAOwBtCtsP/CiJhReD8X6NJSw5I2I4Xa7fL+lT5048NwtzgkRsQ8SR/ZBswrtNU9f59YdS6VsloDZ/V+NZ1PE/YGTiMF6RnAFRFxUTP7bwJMqryJiLmS3mpl22ZmZis8T6l3DFOBHpLqCtt65+0ABwADgX2BtSNiLeB5oI5P7nLgPeCLEdEV+Ebe3tq6K33uWdjWu6rsPVJgrthoCdtYtCQ7R8TzEdEPWB84Ehgqaddm6nqDQv8lVYd7MzMzWwIOnB3DPcCngNMkrSppS2AQUJlG7wosAGYAnSQdTlqr2Ra6kkYPZ0laj7Q2s9XyGssHgAslrSVpbeBCYFxhOv0Z4ABJnSV1A85cwmamk24a6tXSjvl6HippvTyCPJMUMhcW6tq86rDRwMmSNpP0aeB8/G/FzMys1fw/0Q4gImaTbvjZHXiTdFf3dcDP8y7XAk8Br5BG37Ym3ezSFn5EulP7n7nOu9ugzoNJo5h/AyYAs0jrIivOIAW+aaT1lDctSeUR8TJpHenTkmZJOqSFQ/qR1pDOAe4CfhIRj+SyEYByPS/mbUNJn8GTpKUBU2jd2lEzMzMD6hoaGlrey8xqUnfBAv+DamcNJ3lpupnZUlTTEjyPcJqZmZlZqTwUsJyTtCPpQemNOS8izlua/SmDpIOAkU0UHxkR1y/N/piZmdlHeUrdrA2NHTu2ob6+vr27YWZmtrR4St3MzMzM2p8Dp5mZmZmVyoHTzMzMzErlwGlmZmZmpXLgNDMzM7NSOXCamZmZWakcOM3MzMysVH4Op1kbWpp/2tJ/wtHMzDoAP4fTzMzMzNqfA6eZmZmZlcqB08zMzMxK5cBpZmZmZqVy4DQzMzOzUjlwmpmZmVmpHDhtMUnjJJ3S3v3oaCRdJWlUe/fDzMxsWeUH+dliEbFHe/dhaZI0GPhmROze3n0xMzNbnnmE08zMzMxK5RFOW0zSeOAhYAwwEegP/BjoDjwBHBoR0/K+nYHBwD5AN2AqcGREPCppdWBoLvs08BhwXERMKbTzLNAL2B14C/gf0l8ruAjYFPgd0D8i3svHrAucD/QBVgMeBn4YEW+2cE49gZHAdkBDPq8DgC8BpwGdJM3Ju38xIl6TdDhwej6vO3O/FizBpTQzM7MCj3Bac/oBOwEbA2sAQwplV5NC3G5AV+DbwLRcNgLYPn/1AN4GxkpaqXD8IcDPgLWAm4HRpNC5E9AT2BI4DkBSHfBbUmD8fK7zPeCGGs7hPGAKsAGwHjAAmBkRN+ey8RHROX+9JmlH4DLgKGAd4MF8HczMzKyVPMJpzTk7It4GkHQDMDC/Xh/4b+DzETEx7/tKLusEHArUR8QbedsJwLvA10gjpQC3RMRTuXwMaSR1eES8m7fdDSjv+5X8tXtE/DuXnwK8LWmTiHi9mXOYD2wI9I6Il4AXWjjn/sBtEfFgfn+dpCNbOMbMzMya4RFOa860wuu5QJf8umf+/nIjx3QDPkWaugYgIuaQps27N1H3vCa2Vdrrlet8U9IsSbOAV4F/kabfm3Ny7stYSdMkXZKXAzRlE2BS1baJjexnZmZmNfIIp7XGpPx9C+CvVWUzgH+TQmll1LMzsD5pnWdrTCYF3nUiYtGSHBgRM0hT88dJ6k1ak3kKcBbQWF1v8GGgruhJPhczMzNbcg6ctsQi4i1JtwG/lDSAFAg3y2WvSLoOOEfSX4FZwIXABODp1jYJPA9cLOknEfGOpG7AbhFxU3MHSuqX250EzCZNsS/MxdOBTSWtGhHz87bRwH35uZuPAPuT1qo6cJqZmbWSp9SttQ4H/kQKZe+RRg43zGU/IoXEP5Ju2PkM8O2IWNhIPS3Ko5p7k+4Wf0bSe8CTwC41HL5t7uMc4EXS3fHDc9mtpFHX6XmqvldEPAL8ELiKtO70v0g3NZmZmVkr1TU0NLR3H8yWG3UXLFhq/6AaTvIEhZmZtbu6WnbyCKeZmZmZlcpDJLZMk7QpH79xqWJMRBy1NPtz15bjqK+vX5pNmpmZdXgOnLZMy3+9qLnHHJmZmVk785S6mZmZmZXKgdPMzMzMSuXAaWZmZmalcuA0MzMzs1I5cJqZmZlZqRw4zczMzKxU/ktDZm2ozL805L8sZGZmHZD/0pCZmZmZtT8HTjMzMzMrlQOnmZmZmZXKgdPMzMzMSuXAaWZmZmalcuAskDRO0int3Y8VwSe91pIOljSpDbvUXFtXSRq1NNoyMzNbHvk5KwURsUet+0oaDzwUEeeW16MlJ2kAcEZEbN7efWnOklzrskgaDHwzInZv776YmZktzzzCWSJJq3Tk+trD8nAOZmZmtmQ8wllQGbUExgATgf7Aj4HuwBPAoRExTdKlwI7ADpJOBd6IiC3ztOsqwAfAt4GbgaMl7QycD2wFTANGRMTIGvvzJ6AnsCtwnqQxwFXAV4BVgReAEyLiGUk7AJcDq0qak6vZKyLGS/o8cCHwZeB94HrgrIj4oIU+VM5pEbA3MAM4JyJGFfbZERgKbA3MBH4J/DwiGiTtkq/pYcDZQDegS/UIsaQvAhcB2+Y6fg0MjYiFufxrud6t8jV5oKXrl4/rCYwEtgMaSJ/rAcCXgNOAToVr9cWIeE3S4cDpua93kh5qu6CW9szMzOzjPMLZvH7ATsDGwBrAEICIOBZ4lBS8OkfEloVj9gPGkcLK/0rqBdwH/ApYFxgADJW0X419OBy4GFgzf+9ECl49gA2BZ4E7JK0SEU8ARwGv5X51zmFzfeAR4I58LjsA/0EK07X4b+B+YB3gSOBXkr4OIGlr4F5geD7nvsCxwCGF41cC9iSFyQ2qK5e0JvAg8HA+p775vE8slI8Dbst9+BHwgxr7fh4wJbe7Hun6z4yIm3PZ+MK1ei2H58tI13Gd3K9+NbZlZmZmjfAIZ/POjoi3ASTdAAys4ZjHcpgBmCfpAODZwojgk5JG5rpuraG+2yLi95X6SOFpSqVQ0hnAccAWwF+bqKM/8HxhVPUNSUOBYeQQ3YInI2JMfv2QpNtJwe1xUvC7NSLuzOUT8ghwf+C6Qh2DImJ2E/X3BeYD50ZEA/CSpGGkwDkc2AuYCwzL5X+UdDVwUA19n08Ksb0j4iXSiHBz+pOu+YOuy0QqAAAgAElEQVT5/XWSjqyhHTMzM2uCA2fzphVezwW61HDMpKr33UnTuEWvkqana/GR+iStB/wc2AVYizTVDWl0sSm9gG9ImlXYVkcaeVziPuT3Xy7UvaukfQrlnYCphfeLqt5X6w5MzmGy4tW8HWCTRsqrr2lTTgbOBMZKWoM0SvrjiJjTxP6bAFG1rda2zMzMrBEOnK23qMbtU0nTyUW9aT6ANVffUOAzwHZ5PWkX4J+kANlUvyaT1kv2rbHNaj0bef96oe5fR8QxzRzfUBUWq00FekiqK+xXvEZvNFJe3adGRcQM0gjwcZJ6k9ZkngKcRePX6o1G6u4JvFJLe2ZmZvZxDpytNx2o5dFDNwJnSuoP3EAaGTwSOLqV7XYlTa3PlNSZNC1e3a/1JXWNiH/mbdeR1pMenvswnxSiPhsR99XQ5vZ5acAtwM7AvkDlUUK/BB6RdB9prWoD8FmgW0Q8UuM53UO6Yeg0ScNJo6aDSDf7ANxNWr96sqQRwBeA7wP/bqliSf2Ap0mjsrNJ574wF08HNpW0akTMz9tGA/flm6UeAfYn3XDkwGlmZtZKvmmo9UYAkjRL0otN7RQRE0kjnMcC75ACzZkRcUsr2z0LWD/X9QJpHeXCQvnDpBtdJua+7RwR04FvAd8hBa+ZwG9Io4i1uCWfw0zgauCYiPi/fH5/Ia2xPIG0BOEtYBTNT/F/RF7b2YcUYt8k3aB0HWnpABExi7TOs1/uw8Wkm7BqsS0pOM4BXiTdZDU8l91KGkWdnq9VrxySf0h6EsC7wH+RnjZgZmZmrVTX0NDcTKet6PJI34KIqOWGqRVe3QULSvsH1XCSJyTMzKzDqWt5F49wmpmZmVnJPGTSjiSdRnr4eGP2iIhHl0IfDuLDtZLVOvzjgCRtStOPgxoTEUctzf6YmZnZx3lK3awNjR07tqG+vr69u2FmZra0eErdzMzMzNqfA6eZmZmZlcqB08zMzMxK5cBpZmZmZqVy4DQzMzOzUjlwmpmZmVmpHDjNzMzMrFQOnGZmZmZWKj/43awNtfXfUvffTzczsw7OD343MzMzs/bnwGlmZmZmpXLgNDMzM7NSOXCamZmZWakcODNJcyTt0N79WFF80ust6SpJo9qwS8219YqkAUujLTMzs+WRb4HNIqLzkuwvqQHYMSIeK6lLrZJD2IKIGNjefWnOkl7vMkgaDzwUEee2d1/MzMyWZx7hLJGkVdqwrjpJy/wvCG15TczMzGzZsMwFGEmTgCuA3YDtgEnA/0TE4zmQnQYMANYGngWOj4i/1FDv4hHLPH16BnAxcAqwBnAL8IOIWCjp+XzYA5IWATdFxMDct18D3wK+CgwEbpJ0NHACsCHwEnByRDxaY59OAA4BPgd8S9IawHnAZ4EFwO+A4yLiLUmnAAflY/fP1ayZ+/wd4ExgM2AacG5EXF9DHyrn1Af4EjABODoi/ljY5wjgeKA78BowKCIeyGWDgZ1In8Uh+fse1SPEkvYFzgJ6kj7TwRHxm0IbhwOnA92AO0nP/VpQQ/93B4bn854P/Ckidpd0KbAjsIOkU4E3ImLLHIiHAQcDi4ARLbVhZmZmzVtWRzgPB44D1gQeBK7N208G+gN7ksLdo8CDkrq2oo0ewAakoPJVYD9gf4CI2Cbv0yciOldNXx8BnAh0Ae6UdABwTu7XusCVwH2SetTYj+8D/YDOwHPAv4FjScHrC8BGwC9yv84Hrgeuzf3qnMPmfwBXk8LrOsChwKWSdqqxD0eRAuU6wG3AvZVrmsPmIFLQXZsUCu+QtHnh+J1IIbc7sG915ZK+nvt9KukanQbcKGm7XL4jcFnuxzqkz7xfjX2/jvSLw5rAxsC5ABFxLOnn45x8nbbM+58K7AV8HehFCsC1flZmZmbWiGVuhDMbGREvQrp5BDhB0prAYcCwiJiQy4aQRhn7AjcuYRvvA2dFxELgFUm/A0QKRs25MiKeq9Qh6bDc36fytqslDQQOBIbW0I8LIuLV/HohUFwzOl3S+aQRyOYcD/yiMKr6tKQxpBD8hxr6cHVEPAMgaRjwA1IouyHXPSQiKqO+90p6mBTOK2sjJ0fEhfn1/EbqHwDcHhHj8vt7JP2G9IvFU7mft0XEg7n8OklH1tDvSnubARtExHRgfAv79wd+FhGv5PM9iRT6zczMrJWW1cA5rfB6bv7ehTSCNrFSEBGL8pRw91a08VYOm8V2utRw3KSq991J0/FFry5Bnz5Sn6SvkKbUtwFWJ00tt3QDTi/SdPyJhW0rkUb4lqgPEdEgaQqwSaHuyyRdXNh/ZeD1wvvJLdTfHXimaturwJfz602AqCqfSG32Jo2Y/lnSDOCKiLiomf034aPnO1fSWzW2ZWZmZo1YVgNnU6aSpkABkNQpv59aQltN/c3sRc31KesNjK2xner6biJNa+8XEf+UtFdVXdX7Qwp8oyJieI1tVutZeSGpDtiUDwPlZOAnEXFrM8c31qeipq5R5XN7o5HynsArLdRLHnntl/v9TdK62xci4vdN9OsjbeU1s91aasfMzMyatrwFzlHAKZL+QBqlGkQ6x3tKaGs6sAUfneJuqk+/kHQXH9448yXggFa22xWYDbwnaVPSmsPqfm0vqVNEVALVRcAoSU8Cj5NGN78A1EVE9chhYw7PU9x/Bn5EGlmtXNMRwGBJfweeB1YDvgK8XVnaUINrgYckjQYeIt2gtA+wSy4fTVr3Ogp4hDRdvx0tBE5Jq5Ku8z0R8bakmaSQWRm5ng5sXnXYaODk/MikfwDns+yudTYzM+sQlrf/kQ4nrdV8AHgT2JV0Y88/S2jrdGCIpJmSRja1U0TcAJwNjAHeAY4G9oyIlqaZm/I/pHWp7wF3ANUji1eR7qp/R9IsSSvlO8aPIF2ft0lLEkbQ8lR8xRWkG29mkm7W6RsRs/P5XUkKZdfk8imku+FrfvxRRPwf6UamC3Id5wMHR8STufwR4If53N4F/gu4ucbq+wETJM0B7iKNxj6Sy0YAytfpxbxtKHA/8CRp2n4KLS8JMDMzs2bUNTQ0NTNstvixSGdExJj27suyoO6CBW36D6rhpOVtEsLMzJYzdbXstLyNcJqZmZlZB7PCDJ/kKdPGnqc4OSI+t7T7AyBpHOnh4x+ztP70o6TLSQ85b8zWS6MPn4Skg4CmljQcWcvD7c3MzKxcnlI3a0OeUjczsxVMTVPq/r+ZWRu6a8tx1NfXt3c3zMzMOhSv4TQzMzOzUjlwmpmZmVmpHDjNzMzMrFQOnGZmZmZWKgdOMzMzMyuVA6eZmZmZlcqB08zMzMxK5Qe/m7Whtnjwux/2bmZmyxD/LXUzMzMza38OnGZmZmZWKgdOMzMzMyuVA6eZmZmZlcqB08zMzMxK5cBpZmZmZqXy81es1SSNBx6KiHPbuy8Vkn4KHAisC/wL+ANwYkRMKeyzGXABsGve9BKwY0R8kMsF/BL4PDAN+ElEjFlqJ2FmZrac8QintRtJq5RQ7WjgSxHRFegJTAFuKrTZDXgUeB7YFFgHOBZYmMvXBMYBtwNrA0cBl0vaoYS+mpmZrRA8wmmtIulSYEdgB0mnAm9ExJaSjgCOB7oDrwGDIuKBfMxgYCfgWeAQ4FlJRwMTgQHAIKAH8AhwUH5/OLAIOCciLmupXxExofC2Lh+7ZWHbicCUiBhcPKzweh9gHnB+RDQAD0r6DfA/wBMttW9mZmYf5xFOa5WIOJY0UnhORHQuhM1BpLC4NnA6cIekzQuH7kSapu4O7FvYvi/wTdKoY0/gKeBVYCPgMOAiSZvW0jdJB0qaDcwhhd/BheJvAVMl3SPpXUkvSDqoUL4N8FwOmxXP5u1mZmbWCh7htLZ0PDAkIp7P7++V9DCwP1BZ5zk5Ii7Mr+en5ZJACq7vAki6G+gbEVfmsnGSZgLbkqbImxURNwA3SNoQ+D7w50LxesBXgX7A3qQAOlbS5Ih4DOgCzK6qchbQtcWzNzMzs0Y5cFpb6gVcJuniwraVgdcL7yc3cey0wut5Ve8r27osSWciYrqkK4HXJG2aA+17wBMRcVve7UFJ9wHfBh7L5T2rqloL+OeStG1mZmYfcuC0T2JR1fvJpDu6b12CY8q2MrAGaWr+XeBPwOaN7FeZQn8e+E5V2bZ5u5mZmbWCA6d9EtP5aHgbAQyW9HdSQFsN+ArwdtXNPKWQ1An4AXBLRLwlaRPgEmASUGl/JPCopO8AdwE7A32AYbn8N8D5kk4GfkG6MWof4D/K7r+ZmdnyyjcN2ScxgvTYylmSXsxrLs8HrgFmktZbngmU8fijpuwJ/EXSXNKNR/OA3SNiAUBEPEl6Tucw0vT5JcChEfFELp+V69iPtJbzSuCoSrmZmZktubqGhoaW9zKzmtRdsOAT/4NqOMkTD2Zmtsyoq2Unj3CamZmZWak8lGLLFEmXAwc3Ubx18U9YmpmZWcfgKXWzNjR27NiG+vr69u6GmZnZ0uIpdTMzMzNrfw6cZmZmZlYqB04zMzMzK5UDp5mZmZmVyoHTzMzMzErlwGlmZmZmpXLgNDMzM7NS+TmcZm2oqT9t6T9XaWZmyyk/h9PMzMzM2p8Dp5mZmZmVyoHTzMzMzErlwGlmZmZmpfKdDLbUSDoV+BGwBvCtiPjjUmr3cmBBRBy7NNozMzOzj/Jd6rZUSNoEmAJ8PiL+2t79KYvvUjczsxWM71K3DqUnsOiThE1Jq7Rdd8zMzGxp8QinfYSkzsBgYB+gGzAVOBJ4Bhiat38aeAw4LiKm5OPG5316An2At4ATI+JOSf2AUcBqwFzgzYjYTNK6wIi8P8D9wI8i4t1c5yTg18C3gK8CA4GtgB2BAA4n/dL0U+B24Jq838vAwRHxUq5nFGlKfWB+3wAcAxyW63sRGBARE3J5F+AyYC/gPeDM3I/dI2J8c9fPI5xmZraC8QintcrVwHbAbkBX4NvANFIw3D5/9QDeBsZKWqlw7KHAhcCawKXAtZJWj4ibgT2AhRHROSI2y/tfD6wN/L/8tR4wuqo/RwAnAl2AO/O2nYC/AxsCBwPDc7+PAdYBXgIubuE8BwD75janApcUyn4B9CaF0S8AfYGVMDMzs1bxsIstJml94L9J6ywn5s2vSOpECpP1EfFG3vcE4F3ga8ATed+bI+LxXH4F8HNgC+D5RtraCPhP4LMRMTNvOxGYIOkzETEt73plRDyXX78vCeDliLgqbxsn6R3g/sKI5g2kMNuc4YXR2VHAmPx6JeAgYI+IeCtvOy1fFzMzM2sFj3BaUc/8/eWq7d2ATwGVEEpEzCFNm3cv7DetUD43v+zSRFuV4yYWtr1aVQYwqZFjp1W9n1e1bV4z7TZWx9zC/usBqwKTC+XF12ZmZraEHDitaFL+vkXV9hnAv/kwkFbWeq5Pmo5ujcpxPQvbeleVASxqZf2t9TYwn7RsoGLTpdwHMzOz5Yqn1G2xiHhL0m3ALyUNII3sVdZbXgecI+mvwCzSWs0JwNOtbOsfkh4ALpR0KGnR8YXAuMJ0+lIXEQvzlPxgSX8G/kW6KcnMzMxaySOcVu1w4E/AI6Q7tO8k3ZzzI9Kd4X8kPU/zM8C3I2LhJ2jr4NzG30jhdRbQ/xPU11aOJ53jy8BfgAeBBtIor5mZmS0hPxbJrAWStiQF4o0j4h/N7evHIpmZ2Qqmpsci+f+CZlUk9SaN6j5FuoloBPCHlsKmmZmZNc5T6mYftxpwBTAb+DPprvcD27VHZmZmyzCPcJpVyX9+8/Pt3Q8zM7PlhQOnWRu6a8tx1NfXt3c3zMzMOhRPqZuZmZlZqRw4zczMzKxUDpxmZmZmVioHTjMzMzMrlQOnmZmZmZXKgdPMzMzMSuXAaWZmZmalcuA0MzMzs1I5cJqZmZlZqRw4zczMzKxUDpxmZmZmVioHTjMzMzMrVasCp6RNJDVI6lnDvoMlPdSadpYGSZdLurS9+9GRtec1aou2JR0k6fm26pOZmZktmZXbuwM5tE4EukfE6zXsPwA4IyI2b4v2I+KotqhneSFpEun6jqlsa89r1BZtR8T1wPVt0B0zMzNrBU+pryAkrdLefTAzM7MVU00jnJI2BK4AdgbeBM6vKj8COB7oDrwGDIqIBwq71EkaAfQH3gcujYif5bLKVOffJDUAwyLinCb6sQNwObCqpDl5814RMV7SzrlfWwHTgBERMbKGcxsFLIiIgfl9A3AMcFiu60VgQERMyOWrACcDhwIbAW/l871N0srAacAAYG3gWeD4iPhLoa2VgA+AfYC5wEnAS8CVub0ADoqIf+RjVgeGAPsCawJPA8dGxCstnNcuwEP5PM4GugFdJB0PHA1sDMwkjfydERELJY0FNgWuknQ58HhE9GnkGvUALga+Qfo8bwd+HBHvt9CnwcCO+RwPJ/3C89N8/DXAV4GXgYMj4qXCNVsQEQMl1QHn5nPqArwDXBgRl0ham/Qzuivp5/p14KiIeLR6VFzSeOAZoCfQh/QZnhgRd+byOuDHwA+A1YFrgS8Cj0bE4ObO0czMzD6u1hHO64GFpDCyEylQAYvD5iDgIFLIOh24Q1JxynsnUlD9DLA3cKKkA3PZNvn7lhHRuamwCRARTwBHAa/lfTvnsNkLuA/4FbBu7t9QSfvVeH7VBpAC3nrAVOCSQtm5wMHAfkBXUgh/OZedTArVewIbAo8CD0rqWjj+e6SAtQ5wDiloDgG+C2wANJACYkUliG6f63wKuLvGEcuVcl+2zXVDCmJ75L7vTQp+AwEioh6YAgzM17ZPdYU5VN8DTAd65H59A7ighv5A+ln4ez6Xg4HhwNWkkL8OKXxf3MSx/0EK+ttFRBfga8BjuexkUjjsAaxFup7NLdE4FLiQFOIvBa7N4R7gENIvUPWk6zYt99vMzMxaocURTkkbk0aNNo+I2cBsSWcDlRHM44EhEVEZqbxX0sPA/qRwBul/2MMiogF4RtIVpFB3QxudxwHAsxExKr9/UtJIUpC6tRX1DY+IKbB4hG1Mfl1HCkb9IuKFvO/rfBhsDiOdZ2U0dEjuQ1/gxrzP7yPinlx+HSkkj66sX5V0G3BEfr0ecCDQIyLezNvOBk4AtuPDsNWcQflzAyAibi+UPSdpNLAb0OJocPY1YAtS6JsLzJV0BvBbScfmz7g5L0fEVfn1OEnvAPcXRjRvoOn1lvOB1YDPSZoREW+RRicrZesCWwLPRcTLTdRRcXNEPJ7bvAL4eT6v50m/NIyMiOdy+XDS525mZmatUMsI5yb5++TCtomF172AyyTNqnwB3yJN2VZMrgoikwr1toXuVX0CeDVvb41phddzSdO3kKal1+DDEc1m+xERi0jnWuzHtEL5vEbam1dor1f+/kLh2r4LrEJt57aINEK7mKQDJP1R0juSZpOCVLca6qroDszIYbPiVVIQrKWeaVXv59H0+X9ERIwnLVk4A3hL0gOSlIuHA78jTX/PkHStpA0aq6e6H4VzqbS7MYWf9/yz+5HraGZmZrWrZQ3nG/l7D1KwgLT2rWIy8JOIaG4ksYekukLo7MmHo4KLauvqYo3tP5U0dVzUm7YPCTNIgWgL0rRwY/3oWXkjqVN+39p+VELPFhExoxXHNxSDvqTupNHafYBxETFf0gWACse09HlMBbpJWr0QmHsD/yJdn1JFxBXAFXn6ezBwB7BpDo2nA6fnNcdjSCG0fyuaeYP08w4sHtlu7S8vZmZmK7wWA2dEvJ5vsjhf0mHAp4Gz/n97dx5uR1Hmcfz7kgUIIYCSKBJCEgGXAVRS8ODIElA2BUUFEQREFBNHRGREnUA0I8gmyMgzZFhNWBwQGQ0TQAIiPCDPI0wJKsgSCAmBQEyAJCSCQKDmj7cO6XTOOffcm7Pk3vv7PM957u29urq6+z3VVacLs5wPTAkhPI4/jtwAGAc8X3m0jLfdPDl3HNoef2R8Up62GA9ytqV+m7uKhcCIEMKwGONLedw1wOQQwtH4Y/qdgAl455imiTGmEMJUPC/m4x2KtgTelh+xTwe+E0K4C6/Z/C6exzf1cHuL8iPmqSGEE2OMC0IIm+I1yLfFGFd0sYqyoXit9mLg9RDCrnh7xUcK8yzEj0Ut9wFPAOeFEP4Vby95GjCtgcfpayWEsAuwfk7Dq8ByvG0xIYSDcrpmAyvwAPiNHm7qKuDsEMKv8GN8At5BTERERHqg0U5DR+A3+qfxjjBXVibEGC/Fe4dPw3s9zwcm4499K+7Gg86FwI3AT8ntN3PP5snANfmx8SldpOUO4DZgbp5/zxjjXLyG83i85/JVwOQY43UN7l93nAJcB8zAA547gUoHqR/jwe+teCepvYF9C4FxTxwHPAbcGUJYDjyId1jqdnCX20n+ALgBWAp8j1VtSytOB44MISwJIfymyjpWAgfiTSLm48HfvXhv+1Ybiped5/HjvC9wWJ72bmAm8BIe7L+CB/w9cSVwIXAzfhxHAn/Ag1wRERHpJkuppZVSIr1ebhrxNHByjLFuR7eZM2emgw46qD0JExER6TxrZKaOv2lIZF0UQvg8Xou9Hv6bnEOANWp8RUREpGvrXMAZQtid2jf2M2KMZ3RzfZPwns3VHBBjvLs761uXFH78vuzuGOMBbU0MzT92HXY8q34q6iHg4zHGJR1Mj4iISK+lR+oiTaRH6iIi0s809Ehd71IXERERkZZSwCkiIiIiLaWAU0RERERaSgGniIiIiLSUAk4RERERaSkFnCIiIiLSUgo4RURERKSlFHCKiIiISEsp4BQRERGRllLAKSIiIiItpYBTRERERFpKAaeIiIiItJQCThERERFpKQWcIiIiItJSCjhFREREpKUUcIqIiIhISyngFBEREZGWUsApIiIiIi1lKaVOp0Gkz1h//fUfeu211/7R6XT0NwMHDtx85cqVz3c6Hf2R8r4zlO+do7xfw/Mppf27mmlgO1Ii0l/ssMMO/4gxhk6no78JIUTle2co7ztD+d45yvue0SN1EREREWkpBZwiIiIi0lIKOEWa65JOJ6CfUr53jvK+M5TvnaO87wF1GhIRERGRllINp4iIiIi0lAJOEREREWkp/SySSDeFELYDrgDeDrwAHB1jfLw0zwDgAmB/IAFnxRgva3da+5IG830y8HngDeB1YFKMcVa709rXNJL3hXnfAzwATI0xfrt9qex7Gs33EMLngMmA4debj8UY/9bOtPY1DV5vRgDTgK2AQcAdwAkxxpVtTm6voBpOke67CLgwxrgdcCFwcZV5vgBsA2wLfBiYEkIY3bYU9k2N5Pt9wM4xxh2BY4FfhBA2bGMa+6pG8r7yRetiYEYb09aXdZnvIYQATAH2iTFuD+wGLGtnIvuoRsr8JOCRfL3ZERgHfKZ9SexdFHCKdEP+RrsTcE0edQ2wUwhheGnWw4BLY4xvxhgX4zfgQ9uX0r6l0XyPMc6KMb6cB/+C1/i8vW0J7YO6UeYBvgfcCMxuU/L6rG7k+7eAc2OMCwFijMtijHrb2VroRt4nYOMQwnrA+sBgYEHbEtrLKOAU6Z6tgAUxxjcA8t9n8/iiUcBTheH5VeaRxjWa70VHA3NijM+0IX19WUN5H0L4ALAfcH7bU9g3NVrm3w+MDSHcFUK4P4RwagjB2pzWvqbRvD8N2A54DlgIzIox3tPOhPYmCjhFpM8JIeyJ3wwO73Ra+oMQwiD8twknVm7S0jYD8Me5+wB7AgcAR3U0Rf3HofiTlC2ALYE9QgiHdDZJ6y4FnCLd8zSwZW6rVmmz9q48vmg+sHVheFSVeaRxjeY7IYQPA1cDB8cYH2trKvumRvJ+C+DdwM0hhHnAicBxIQT9QHbPdedac32M8dUY43LgBmCXtqa072k0778B/Dw3nVqG5/1ebU1pL6KAU6QbYoyLgD+xqubscOCB3E6z6Jf4DXe93O7nYOD69qW0b2k030MIOwO/AA6JMd7f3lT2TY3kfYxxfoxx8xjj6BjjaOA/8DbMX217gvuIblxr/hvYN4Rguab5o8Cf25fSvqcbeT8X/yUSQgiDgY8BD7Urnb2NAk6R7psIfCOEMBv/hjsRIIRwc+4xCnAV8CTwOPAH4IcxxrmdSGwf0ki+TwU2BC4OIfwpf3boTHL7lEbyXpqvkXy/FlgEPIwHSX8FLu9AWvuaRvL+RGD3EMKDeN7PBi7tRGJ7A73aUkRERERaSjWcIiIiItJSCjhFREREpKUUcIqIiIhISyngFBEREZGWUsApIiIiIi2lgFOkHzOz/czs7sLweDOb18EktY2ZTTezy5q4vtFmlgrDw83sKTPbvIFlJ5rZVc1KS29gZrub2dJOp6M/MrMju3OeN/tckfpadW704LifZWanNWv7CjhF+ikzM/y91z/oYr6vmdlDZvaSmS0xs2hmhxWmzzOzI6sst8Z4c7PzuoaWpo03s2RmK/LnWTObZmZvW7s97YyU0mL8R7m7yt+NgB8CU9qQrHVGSunulNKmnU5HLWY2xcx+2+l09Aetymszu9PMTm32elutfG50sCyeDXzdzLZsxsoUcIr0X/sCg4E7as1gZofjAdOXgU3w17t9C1jSw23uBYwF3qT6e87fSCkNTSkNBXYDPoy/taa3+hnwJTMbVmeeI4EHU0pz2pSm1ZjZADPTvUBEVpNSWgL8BpjQjPXpIiPSBrm271QzuyPX3j1oZjua2eFm9oSZLTOzy8xsYGGZUWZ2vZktNLPnzOwSM9u4MP0MM3syr2+OmZ1YmDY61xYeZWYPm9lyM7vVzLYoJOtg4Lep/tsf/hm4K6V0b3Kv5G/ft/YwKyYAt+BvYqp7EUspPQncCHyoPM3MBuY8Obg0frqZTcv/f9TM7s21sovN7FozG1Frezm/disMjzezlaVtTso1tEvN7B4zq/uWnZTS48Dz+CvvajkYuK2Ulm+a2aP5uM03szPNbECe9mMzm1Gaf3yed6M8vL2Zzcr7XVl+UJ5WKRtfNrOHgZeBEWb2eTP7c659fs7MLq6sLy/3TjObmYOpV4wAAAukSURBVMvq7Lx8MrPRhXmOy7Xhy8zsATPbt9ZOV8nf6WZ2lZn9LOfvgnx+fNDM/i/v3x1m9q7CMvPM7Ptm9vt8HkQz27kwvW4ZMLNB+Zg+ltc/x8wOMa/BnwSMt1U17mNr7MeeeRvL8jGbUJg23sxWmtlhed3LzOy64nlcZX09uVbsaGa/y/v5ZF5+QGH6LjlvVpjZ7/EvfcVtDjGzc81srpm9aGa3mNk2tdJYJc1vN7Mrza9VC83sCis8mbDS045CGRxZK6/N7Ji8v9/N5XGRmZ1XpRyPLKz3GDN7Iv//n8DuwOS8zsdqpH2Kmd1uZmfnMvKCmZ1kZlvnPF1uZn80s/cVllmrc6VQ1i8tlPU1yk3+v27+lPZltaYPTTrut+HXqLWXUtJHH31a/AHm4a+5fB8wCLgamANcAmwEjMJfT/eFPP8GwBP4o9YNgc2Am4GfFdZ5JF7jaMDewCvAfnnaaCDhAdvmwDDgHuDSwvL3AieU0jkemFcYPhT4B3A6/o7mTWvs25FdjQeGA68Cn8GDyASMK217ZWF4G+Cx4j6X1n8OMKMwPBRYAeyeh3cDdgYGAu8E7gKuKcw/HbisMJyA3eqk50c5z8YCA/Ba3+eBzYp5XiWdM4HT65SNvwGfLI37LDAmH9sP5Xkm5GnvB14DhhfmvwK4PP8/AngBD+gHA1sCEfh+qWzcnvNlcN6fA4B/wisitsFflXhmYRu3A/+Ty9II4M68ntF5+nF4mf1AXsfH8/HYpsZ+l/N3Ol6GP5GXn5iX/19gJDAE+B2rl+F5wLPAuLwf3wMWA8MaLANn5/3cMef1SGDHPG0K/oWs3nk9Jqf5mLyNXYEXgUML+5jwV00OBd6BXwdOaeK1YpNcPiYD6+flngROLkx/IefN4JwfC1n9PP85fq14R57n34FHgUHVzpUqab4FL+eb5c9NwE11rgWjc76MrJXXOU9fBy7Er4Hvxl8dOanaOgrLPFEYvhM4tYtjOCVv5yusOg/eAH5bOga3FZZZ23NlOl5uPpnX8Zmchq1rnBu18ueJ0ri3jlMzjnueZxz+RGpwvXxs5NPWm64++vTXT77gnlwY/ni+ABWDhuuA8/P/hwBzSusYhwdsA2ps43rgnPx/5WK8c2H614EHCsOzgWNK6xhfvCDlcQcCv8Jvam/gj+C3L+3b34Glpc+brH6T+Q5+o6zcxO4HLi5tO+VllwBzgYuoEuTm+d+HB14j8vCxwOw6x+BAYFFh+K2Lcx6uGXDiwchyYI/SOh+s7CO1A86fA1PrpOs1YHwX5edc4LrC8L3At/L/G+f8/0ge/jbwu9LynyXfnAplY48utnk8cF/+f2ReZmxh+kdZ/Sb6EHB0aR0zqXHDp3rAWQxShuT1H1oY9y+sXobnAacVhg2YDxzRVRnI864APlFj3il0HXBOAu4pjTsTmFUq08Xz/MfAr+uscx7du1YcATxNflV1HjcBeCz//4WcJ8XpPyKf5/gX0gSMKkxfD1hGPh+oE3DiX3oTsG1h3HvyuC0K+9STgPNVYEhh3FfI53h5HYVlehJw/rU0blGVY7CkiefKdAplPY9bDHyqxrlRK3/qBZxrfdzzuG3zfCPq5WMjn7eq5EWk5Z4r/P8y3l5xcWlc5VHbGGCUrdlTMeE1NQvM7AS8VmkkfvPcEO+kUmubfy+sHzyoq9e20DeY0o34t2DM7L3AVOBGMxuT8hUJr327uricFXpDmpnltF6dUno9j74cOMvMvp1SWp7HvZEa7EiSUnrEzO7Ha3p/AnwJmFbY5jjgDLzGbQieR0OrrKoRm+dlZ1qhJzpe+zGy+iJvGYYHz7WscRzM286ehNemDsRrH/5QmGUa8DW809fngGdSSvfkaWOAj5TKjuG1N0XzStvcB/g+8F68pmwAfuMFryUFv4FVPFVa3xjgQjO7oDBuIPAMjXurvKaUXvZis8Z5U34cPa+wTDKz+eRj0kUZGI7XGM7uRvrKtmLNYzsH+FRhuHyel8/DarpzrdgKeKpwLlbSsFX+f2SV6cU0j8l//5Lzu2JQYR31VOYprnNOYdpz9NyilNLLheF5dH2+9UQ5jS9Tp9w14Vypts1GykV3NOu4D2NVRcBaURtOkXXTU/g3+U1Lnw1SSgvM7CP448AJwOY5SJuJ31Ab9QD+eLZhKaVH8SBna/zRWaP2xh89HVtp54U/vhmK19D01DTgmNzuaFfgysK0a/Fa1O1SSsOo3kmpaAUegFS8q/D/8/gN4WOl47FRSumsLta7PZ7Xtax2HMxsK/wR3ul4DdEm+GPF4rG9FtjOzHbCazqmFaY9hdeGFNO5SfKOWEVvFrY5GJiR1zsq59d3C9tckP+OKixf/L+y3WNL2x2aUvpanX1vhtGVf/IXm1GsCnLrlYHFeCCxbY31vlljfNHTxe1nY/P4dnka2NpWjxqKaVhQZfrowv+VYGjb0rEbklK6psHtl9c5tjRtObXPLaid1yPMbEgp3ZVjW/mS2pP19liTzpXuqrYf5TyF1fe/Wcd9e7wG+LUepv0tCjhF1k03AoPNOzRsbG5LM/t0nj4Mf7y9GEhm9gm8XVF3zMAf9dRkZsea2aGWf0syN9CfCDycUnqxG9uagLefey/wwfzZHg+UvtrNdBddiweyF+BtrBYUpg3DHw8tN7NReFumev4IfNHMBufG/SdVJuRagp8C55rZtgBmNtT8d0zLN7m35EB4ON4erJYZrN6paCh+bV4MvG5muwJHFRdIKS0Ffo0HpbvibTgrrgRCPnYbmNl6uZPB/nXSMBivqVmSUnrFzN6PPyasbO8Z/PHkWbk8DgfKPzdzPjDFvJOPmdmGZrZbrhVvpWPNbCfzziQn4zWZN+VpNctAPqZTgXPMO1mZeSeWHfMsC/GnDIPrbPsaYJyZHW3eqWwXvKxf3tQ9rO8m/NhNymX3PXgAVEnDjXiZOtm8k9ROePtjAFJKi/AnI1Mt//yNmW1qZp+20k+XVZNSeha4FTgvL7cZcB7wm5RSpRbvj8Dh+ZwZjrc3LaqV1+sBZ+eyNBZvLnJF3u4L5C855r+0sAP+FKW83oY7PzWoGedKd1XLnz/hAfmB+Rz/NLBHYXqzjvs++DVqrSngFFkH5cdIe+M1X4/iN83b8UANYBYeWNyH174dggcg3TELWGlm4+vMswR/dPuImf0dbzu4FG8L1xDzXsEHA+emlBYWP3gt7Yesi97etaSUluH7fQD+E0RFX8XbfC3H26D+sovVHY/fnF7E28hNL03/AXADcIOZvYR37JhI/evoscD0nM5argI+kG+opJQeKWxrKR4kVatpmobv96zCjZ2cr3vheT4PP4a/ptRDtSiltAI/zueY2Qq8RrXcPOMIPJh7Bu+AVsnPV/M6LsU7ck3L25yPBxaD6ux7M1yCf+FYAhyGt8ms5HdXZeAU/FjPyPPcyaoA5Zd4Dd1C857EY0rLklKai7fvOx7voHEVMDmldF2zdq4reV/3xb+0/I1V14af5OlL8Y5Yh+F5dAHwX6XVHId30LvTzJbjbZMPxR+lNuJIPP8ew69XS4GjC9NPxb8gP4fn8bWl5Wvl9VN4eZuLX3tuwctYxRfxa9GyvL/lQP98/MvXUjP7a4P7UlczzpUeWCN/kv+M2jfx8v8isD/eUamSzrU+7ma2KV6+L+phuldjqz/eF5H+JNd6TUop7ZGHx+MB0uhOpqs3yrWic1NKloeH473DQ6n9XbVlJ+Kdfo6qN9+6xMz2w4PiDVOHbiTm7YRPLbcflt7PzI7Bj22zayjbbl04V3rCzM7E2w835cfz1WlIpB9LKd2C1xpIk+Ugc+sG572IJtUitIqZfRBvS/Yg3uHgdOAXvekGKtIOfeVcSSn9WzPXp0fqIlI0j979Zp9OWop3hOqrNsMfS68Afg/8BX+kJyKr07lShR6pi4iIiEhLqYZTRERERFpKAaeIiIiItJQCThERERFpKQWcIiIiItJSCjhFREREpKX+H44yDQk3Jym1AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x396 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "shap.summary_plot(shap_values_gbm, X, plot_type=\"bar\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/navdeep/article-information-2019/models/GBM_BEST_GRID_v2_HMDA_1575927435.3138359/gbm_grid2_model_20\n"
     ]
    }
   ],
   "source": [
    "# Save the model\n",
    "model_path = h2o.save_model(model=best_gbm2, path=\"../models/GBM_BEST_GRID_v2_HMDA_\" + str(time.time()), \n",
    "                            force=True)\n",
    "\n",
    "print(model_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Repeat Previous but with XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "xgboost Grid Build progress: |████████████████████████████████████████████| 100%\n"
     ]
    }
   ],
   "source": [
    "# Train and validate a random grid of XGBoost models\n",
    "mxgb_grid = H2OGridSearch(model=H2OXGBoostEstimator(distribution='bernoulli',\n",
    "                    # monotonicity constraints\n",
    "                    monotone_constraints=mono_constraints,\n",
    "                    # more trees is better if the learning rate is small enough \n",
    "                    # here, use \"more than enough\" trees - we have early stopping\n",
    "                    ntrees=10000,\n",
    "                    # learning rate annealing: learning_rate shrinks by 1% after every tree \n",
    "                    # (use 1.00 to disable, but then lower the learning_rate)\n",
    "                    # learn_rate_annealing = 0.99,\n",
    "                    # score every 10 trees to make early stopping reproducible \n",
    "                    # (it depends on the scoring interval)\n",
    "                    score_tree_interval = 10,\n",
    "                    # fix a random number generator seed for reproducibility\n",
    "                    seed = 1234,\n",
    "                    # early stopping once the validation AUC doesn't improve by at least 0.01% for \n",
    "                    # 5 consecutive scoring events\n",
    "                    stopping_rounds = 5,\n",
    "                    stopping_metric = \"AUC\",\n",
    "                    stopping_tolerance = 1e-4),\n",
    "                    grid_id='mxgb_grid2',\n",
    "                    hyper_params=hyper_params,\n",
    "                    search_criteria=search_criteria)\n",
    "mxgb_grid.train(x=X, y=Y,\n",
    "                training_frame=train,\n",
    "                fold_column=fold_column,\n",
    "                ntrees=100,\n",
    "                seed=SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get the grid results, sorted by validation AUC\n",
    "mxgb_gridperf = mxgb_grid.get_grid(sort_by='auc', decreasing=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "          col_sample_rate learn_rate max_depth         sample_rate  \\\n",
      "0     0.30000000000000004        0.1         6  0.7000000000000001   \n",
      "1      0.7000000000000001       0.09        10                 0.5   \n",
      "2                     0.8       0.09        10  0.7000000000000001   \n",
      "3     0.30000000000000004       0.08         6                 0.8   \n",
      "4                     0.5        0.1         8  0.7000000000000001   \n",
      "5      0.7000000000000001       0.08         9  0.6000000000000001   \n",
      "6                     0.2       0.06         7                 0.8   \n",
      "7                     0.2       0.09         6                 1.0   \n",
      "8                     0.4       0.08         5                 0.8   \n",
      "9      0.7000000000000001        0.1         3  0.7000000000000001   \n",
      "10                    0.2        0.1         3                 0.8   \n",
      "11                    0.4       0.07         5                 0.5   \n",
      "12    0.30000000000000004       0.04        10                 0.9   \n",
      "13                    0.4       0.04         9                 0.5   \n",
      "14                    1.0       0.09        10                 0.9   \n",
      "15                    0.8       0.05        10  0.6000000000000001   \n",
      "16                    0.2       0.06         4                 0.9   \n",
      "17                    1.0       0.09         5                 0.5   \n",
      "18     0.7000000000000001       0.08         2                 0.5   \n",
      "19                    1.0       0.07         2  0.6000000000000001   \n",
      "20     0.6000000000000001       0.04         7                 0.8   \n",
      "21                    0.9       0.08         6                 0.8   \n",
      "22     0.7000000000000001       0.04         8                 1.0   \n",
      "23                    0.9       0.06         3                 0.8   \n",
      "24                    0.9       0.06         6                 0.9   \n",
      "25                    0.9       0.04         7                 1.0   \n",
      "26                    0.1        0.1        10                 0.5   \n",
      "27     0.6000000000000001       0.05         4                 0.9   \n",
      "28                    0.4       0.01         6  0.6000000000000001   \n",
      "29                    0.9       0.04         4                 0.5   \n",
      "30                    0.2       0.03         7                 1.0   \n",
      "31    0.30000000000000004       0.06         2  0.7000000000000001   \n",
      "32                    0.1       0.09         3                 0.9   \n",
      "33                    0.1       0.02         6                 0.5   \n",
      "34                    0.1       0.01         6  0.6000000000000001   \n",
      "35                    0.1       0.01        10                 0.5   \n",
      "\n",
      "              model_ids                 auc  \n",
      "0   mxgb_grid2_model_12  0.8013679825930267  \n",
      "1   mxgb_grid2_model_36  0.8005732690837006  \n",
      "2   mxgb_grid2_model_26  0.8001429599667496  \n",
      "3   mxgb_grid2_model_16   0.800061163305402  \n",
      "4   mxgb_grid2_model_20  0.8000163483776875  \n",
      "5   mxgb_grid2_model_27  0.7994088339793088  \n",
      "6   mxgb_grid2_model_31  0.7992874017700462  \n",
      "7   mxgb_grid2_model_22  0.7992674766768099  \n",
      "8    mxgb_grid2_model_3  0.7987344770372624  \n",
      "9   mxgb_grid2_model_18  0.7984855049939724  \n",
      "10  mxgb_grid2_model_25  0.7979467570933864  \n",
      "11   mxgb_grid2_model_6  0.7978859914482165  \n",
      "12   mxgb_grid2_model_2   0.797723764331112  \n",
      "13  mxgb_grid2_model_19  0.7975562064294165  \n",
      "14  mxgb_grid2_model_28  0.7975375039298728  \n",
      "15   mxgb_grid2_model_4   0.797124510177389  \n",
      "16  mxgb_grid2_model_14  0.7969731236433671  \n",
      "17  mxgb_grid2_model_24  0.7969296355114667  \n",
      "18  mxgb_grid2_model_15   0.796761413877883  \n",
      "19   mxgb_grid2_model_5   0.796703267758263  \n",
      "20  mxgb_grid2_model_33  0.7964756736262469  \n",
      "21  mxgb_grid2_model_11  0.7961648696472765  \n",
      "22  mxgb_grid2_model_34  0.7952605911698504  \n",
      "23  mxgb_grid2_model_17  0.7948503278246608  \n",
      "24   mxgb_grid2_model_7  0.7948361977498252  \n",
      "25  mxgb_grid2_model_35  0.7947479897635131  \n",
      "26   mxgb_grid2_model_9  0.7946614394370157  \n",
      "27  mxgb_grid2_model_30  0.7941922504601832  \n",
      "28   mxgb_grid2_model_1  0.7941620661337333  \n",
      "29  mxgb_grid2_model_13  0.7930581615517761  \n",
      "30  mxgb_grid2_model_29  0.7913448798471946  \n",
      "31   mxgb_grid2_model_8  0.7906714239863125  \n",
      "32  mxgb_grid2_model_21   0.789005613918231  \n",
      "33  mxgb_grid2_model_32  0.7782429007690346  \n",
      "34  mxgb_grid2_model_23  0.7749054535522796  \n",
      "35  mxgb_grid2_model_10  0.7719871172868997  \n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(mxgb_gridperf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8076432704169134"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Grab the top GBM model, chosen by validation AUC\n",
    "best_mxgb = mxgb_gridperf.models[0]\n",
    "\n",
    "# Now let's evaluate the model performance on a test set\n",
    "# so we get an honest estimate of top model performance\n",
    "best_mxgb_perf = best_mxgb.model_performance(test)\n",
    "\n",
    "best_mxgb_perf.auc()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Shapley Importance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "First 5 rows of contributions:\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead>\n",
       "<tr><th style=\"text-align: right;\">  debt_to_income_ratio_missing.False</th><th style=\"text-align: right;\">  debt_to_income_ratio_missing.True</th><th style=\"text-align: right;\">  debt_to_income_ratio_missing.missing(NA)</th><th style=\"text-align: right;\">    term_360</th><th style=\"text-align: right;\">  conforming</th><th style=\"text-align: right;\">  loan_amount_std</th><th style=\"text-align: right;\">  loan_to_value_ratio_std</th><th style=\"text-align: right;\">  no_intro_rate_period_std</th><th style=\"text-align: right;\">  intro_rate_period_std</th><th style=\"text-align: right;\">  property_value_std</th><th style=\"text-align: right;\">  income_std</th><th style=\"text-align: right;\">  debt_to_income_ratio_std</th><th style=\"text-align: right;\">  BiasTerm</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td style=\"text-align: right;\">                          -0.0279054</td><td style=\"text-align: right;\">                        -0.017118  </td><td style=\"text-align: right;\">                                         0</td><td style=\"text-align: right;\">-4.30537e-05</td><td style=\"text-align: right;\"> -0.0158831 </td><td style=\"text-align: right;\">      -0.191981  </td><td style=\"text-align: right;\">                -0.39241 </td><td style=\"text-align: right;\">                 -0.104848</td><td style=\"text-align: right;\">              -0.155696</td><td style=\"text-align: right;\">           -0.205218</td><td style=\"text-align: right;\">  -0.0455992</td><td style=\"text-align: right;\">                0.00208951</td><td style=\"text-align: right;\">         0</td></tr>\n",
       "<tr><td style=\"text-align: right;\">                          -0.0428628</td><td style=\"text-align: right;\">                        -0.0220296 </td><td style=\"text-align: right;\">                                         0</td><td style=\"text-align: right;\">-4.53843e-05</td><td style=\"text-align: right;\"> -0.0189243 </td><td style=\"text-align: right;\">      -0.361263  </td><td style=\"text-align: right;\">                -0.931136</td><td style=\"text-align: right;\">                 -0.144612</td><td style=\"text-align: right;\">              -0.250869</td><td style=\"text-align: right;\">           -0.497511</td><td style=\"text-align: right;\">  -0.104077 </td><td style=\"text-align: right;\">               -0.474516  </td><td style=\"text-align: right;\">         0</td></tr>\n",
       "<tr><td style=\"text-align: right;\">                          -0.0110341</td><td style=\"text-align: right;\">                        -0.00667279</td><td style=\"text-align: right;\">                                         0</td><td style=\"text-align: right;\"> 7.18053e-05</td><td style=\"text-align: right;\">  0.00167255</td><td style=\"text-align: right;\">       0.00831683</td><td style=\"text-align: right;\">                -0.419404</td><td style=\"text-align: right;\">                 -0.342049</td><td style=\"text-align: right;\">              -1.13808 </td><td style=\"text-align: right;\">           -0.275266</td><td style=\"text-align: right;\">  -0.0721013</td><td style=\"text-align: right;\">               -0.163921  </td><td style=\"text-align: right;\">         0</td></tr>\n",
       "<tr><td style=\"text-align: right;\">                          -0.013573 </td><td style=\"text-align: right;\">                        -0.0597384 </td><td style=\"text-align: right;\">                                         0</td><td style=\"text-align: right;\">-4.65753e-05</td><td style=\"text-align: right;\"> -0.10398   </td><td style=\"text-align: right;\">      -0.455444  </td><td style=\"text-align: right;\">                -2.39687 </td><td style=\"text-align: right;\">                 -0.434669</td><td style=\"text-align: right;\">              -0.376237</td><td style=\"text-align: right;\">           -0.714399</td><td style=\"text-align: right;\">  -0.0656964</td><td style=\"text-align: right;\">               -0.529831  </td><td style=\"text-align: right;\">         0</td></tr>\n",
       "<tr><td style=\"text-align: right;\">                          -0.0278799</td><td style=\"text-align: right;\">                        -0.0171317 </td><td style=\"text-align: right;\">                                         0</td><td style=\"text-align: right;\">-4.03123e-05</td><td style=\"text-align: right;\"> -0.0108963 </td><td style=\"text-align: right;\">      -0.148726  </td><td style=\"text-align: right;\">                -0.316384</td><td style=\"text-align: right;\">                 -0.102255</td><td style=\"text-align: right;\">              -0.152272</td><td style=\"text-align: right;\">           -0.298293</td><td style=\"text-align: right;\">  -0.052357 </td><td style=\"text-align: right;\">               -0.00910205</td><td style=\"text-align: right;\">         0</td></tr>\n",
       "</tbody>\n",
       "</table>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "shap_values_mxgb, expected_value_mxgb = get_shapley(best_mxgb, train, X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAApwAAAFeCAYAAAAog8+HAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xm8X9O9//HXiaFKEmNQRAaUnw6qfbdoayhu7iVSLXVjjNC4KEVdRE2NUGmERg2tGCok5qEliKkV5Zr6QWlVqshEE4IkzaCNJOf3x1pftq8zfHOcnXNy8n4+Hudxvt+99l5r7f094X3WWnufuvr6eszMzMzMytKprTtgZmZmZh2bA6eZmZmZlcqB08zMzMxK5cBpZmZmZqVy4DQzMzOzUjlwmpmZmVmpHDjNzMzMrFQOnGZmZmZWKgdOMzMzMyvVym3dAbOOZNy4cfX9+vVr626YmZktK3W17OQRTjMzMzMrlQOnmZmZmZXKgdPMzMzMSuXAaWZmZmalcuA0MzMzs1I5cJqZmZlZqRw4zczMzKxUDpxmZmZmVioHTjMzMzMrlQOnmZmZmZXKgdPMzMzMSuXAaWZmZmalcuA0MzMzs1I5cJqZmZlZqRw4zczMzKxUDpxmZmZmVqq6+vr6tu6DWYdRd8Ei/4MyM7N2qf6klcuotq6WnTzCaWZmZmalcuA0MzMzs1I5cJqZmZlZqRw4zczMzKxUDpxmZmZmVioHzgJJ4yWd0tb9WBF80mst6WBJk1uxS021dZWk0cuiLTMzs46olPvjl1cRsUet+0qaADwUEeeW16OlJ2kgcEZEbN7WfWnK0lzrskgaAnwzInZv676YmZl1ZB7hLJGkVdpzfW2hI5yDmZmZLR2PcBZURi2BscAkYADwY6A78ARwaERMl3QpsCOwg6RTgTciYss87boK8D7wbeBm4GhJOwPnA1sB04GRETGqxv78CegJ7AqcJ2kscBXwFWBV4AXghIh4RtIOwOXAqpLm5Wr2iogJkj4PXAh8GXgPuB44KyLeb6YPlXNaAuwNzATOiYjRhX12BIYBWwOzgF8CP4+Iekm75Gt6GHA20A3oUj1CLOmLwEXAtrmOXwPDImJxLv9arnerfE0eaO765eN6AqOA7YB60ud6APAl4DSgU+FafTEiXpN0OHB67uudpIfaLqqlPTMzM/s4j3A2rT+wE7AxsAYwFCAijgUeJQWvzhGxZeGY/YDxpLDyv5J6AfcBvwLWBQYCwyTtV2MfDgcuBtbM3zuRglcPYEPgWeAOSatExBPAUcBruV+dc9hcH3gEuCOfyw7Af5DCdC3+G7gfWAc4EviVpK8DSNoauBcYkc+5L3AscEjh+JWAPUlhcoPqyiWtCTwIPJzPqW8+7xML5eOB23IffgT8oMa+nwdMze2uR7r+syLi5lw2oXCtXsvh+TLSdVwn96t/jW2ZmZlZAzzC2bSzI+JtAEk3AINqOOaxHGYAFkg6AHi2MCL4pKRRua5ba6jvtoj4faU+UniaWimUdAZwHLAF8NdG6hgAPF8YVX1D0jBgODlEN+PJiBibXz8k6XZScHucFPxujYg7c/nEPAI8ALiuUMfgiJjTSP19gYXAuRFRD7wkaTgpcI4A9gLmA8Nz+R8lXQ0cVEPfF5JCbO+IeIk0ItyUAaRr/mB+f52kI2tox8zMzBrhwNm06YXX84EuNRwzuep9d9I0btGrpOnpWnykPknrAT8HdgHWIk11QxpdbEwv4BuSZhe21ZFGHpe6D/n9lwt17yppn0J5J2Ba4f2SqvfVugNTcpiseDVvB9ikgfLqa9qYk4EzgXGS1iCNkv44IuY1sv8mQFRtq7UtMzMza4ADZ8stqXH7NNJ0clFvmg5gTdU3DPgMsF1eT9oF+CcpQDbWrymk9ZJ9a2yzWs8G3r9eqPvXEXFME8fXV4XFatOAHpLqCvsVr9EbDZRX96lBETGTNAJ8nKTepDWZpwBn0fC1eqOBunsCr9TSnpmZmX2cA2fLzQBqefTQjcCZkgYAN5BGBo8Ejm5hu11JU+uzJHUmTYtX92t9SV0j4p9523Wk9aSH5z4sJIWoz0bEfTW0uX1eGnALsDOwL1B5lNAvgUck3Udaq1oPfBboFhGP1HhO95BuGDpN0gjSqOlg0s0+AHeT1q+eLGkk8AXg+8C/m6tYUn/gadKo7BzSuS/OxTOATSWtGhEL87YxwH35ZqlHgP1JNxw5cJqZmbWQbxpquZGAJM2W9GJjO0XEJNII57HAO6RAc2ZE3NLCds8C1s91vUBaR7m4UP4w6UaXSblvO0fEDOBbwHdIwWsW8BvSKGItbsnnMAu4GjgmIv4vn99fSGssTyAtQXgLGE3TU/wfkdd29iGF2DdJNyhdR1o6QETMJq3z7J/7cDHpJqxabEsKjvOAF0k3WY3IZbeSRlFn5GvVK4fkH5KeBPAu8F+kpw2YmZlZC9XV1zc102krujzStygiarlhaoVXd8Ei/4MyM7N2qf6kUia265rfxSOcZmZmZlYyr+FsQ5JOIz18vCF7RMSjy6APB/HhWslq7f5xQJI2pfHHQY2NiKOWZX/MzMzs4zylbtaKPKVuZmbtVVtOqXuE06wV3bXlePr169fW3TAzM2tXvIbTzMzMzErlwGlmZmZmpXLgNDMzM7NSOXCamZmZWakcOM3MzMysVA6cZmZmZlYqB04zMzMzK5Uf/G7Wivzgd+tISnpItJl1LP5b6mZmZmbW9hw4zczMzKxUDpxmZmZmVioHTjMzMzMrlQOnmZmZmZVquQqckjaRVC+pZw37DpH00DLoVotIulzSpW3dj/asPV0jSQ9JGtLW/TAzM1serbDPvMihdRLQPSJer2H/gcAZEbF5a7QfEUe1Rj0dhaTJpOs7trJtWVwjSaOBRRExqOy2zMzMVlTL1QinLX8krdLWfTAzM7O21a5HOCVtCFwB7Ay8CZxfVX4EcDzQHXgNGBwRDxR2qZM0EhgAvAdcGhE/y2XP5+9/k1QPDI+Icxrpxw7A5cCqkublzXtFxARJO+d+bQVMB0ZGxKgazm00hZG13IdjgMNyXS8CAyNiYi5fBTgZOBTYCHgrn+9tklYGTgMGAmsDzwLHR8RfCm2tBLwP7APMB04CXgKuzO0FcFBE/CMfszowFNgXWBN4Gjg2Il5p5rx2AR7K53E20A3oIul44GhgY2AWcD1pRHOxpHHApsBVki4HHo+IPg1cox7AxcA3SJ/n7cCPI+K9Zvq0LXAJ8AVgMTAR6AscARyU99k/774msAQ4lfR5rA5cS40PtjUzM7OPa+8jnNeTAsKmwE6kQAV8EDYHkwLD2sDpwB2SilPeO5GC6meAvYETJR2Yy7bJ37eMiM6NhU2AiHgCOAp4Le/bOYfNXsB9wK+AdXP/hknar4XnO5AU8NYDppFCUsW5wMHAfkBXUgh/OZedTArVewIbAo8CD0rqWjj+e6SAtg5wDiloDgW+C2wA1JMCYkUliG6f63wKuLvGEcuVcl+2zXUDvA7skfu+N3A4MAggIvoBU4FB+dr2qa4wh+p7gBlAj9yvbwAX1NCfy4AH8rlvAJwILIyI80k/Y9cWPtfFpOv8o9zPDYG3ST9LZmZm1gLtdoRT0sbArsDmETEHmCPpbFJwgDSyOTQiKiOV90p6GNifFM4gjTgOj4h64BlJV5BC3Q2t1M0DgGcjYnR+/6SkUaQgdWsL6hsREVPhg1HJsfl1HWm0rX9EvJD3fT1/QRpNHF4YDR2a+9AXuDHv8/uIuCeXX0cKyWMq61cl3UYa8UPSesCBQI+IeDNvOxs4AdgOeKyGcxmcPzcAIuL2QtlzksYAuwHNjgZnXwO2ALaLiPnAfElnAL+VdGz+jBuzkPRLS/eImAw82UxbA4BREfEMgKRhpF84zMzMrAXabeAENsnfpxS2TSq87gVcJuniwraV+TCEAUypCiKTSVPKraV7VZ8AXiWNjLXE9MLr+UCX/LobsAYfjmg22Y+IWJJvwuneUN0RsUBSdXsLCu31yt9fyPtVrFJVZ2OWkEZoPyDpANLIYm/S57QqzQe/ou7AzBw2K14FViNdn7eaOPYw4EzgMUnvk4L82RGxqJH9NyH9rAAfXM8pjexrZmZmzWjPgfON/L0HKVgA9CyUTwF+EhFNjST2kFRXCJ09+TCQLlnK/jS0/zTS1HFRb6rCViuYSQqEWwB/b6QfPStvJHXK71vaj0q42iIiZrbg+Ppi0JfUnRTy9gHGR8RCSRcAxTTb3OcxDegmafWIWJC39Qb+Rbo+jYqISaQpfCR9gTRKPgn4dSPtvsFHr2cd6efQzMzMWqDdBs6IeF3SBOB8SYcBnwbOKuwyEhgi6e+kG4BWA74CvF2ZWiat3Tw53zj0edKU8Ym5bCYpbGzBR0dFGzMDWF9S14j4Z952I3CmpAGkafovA0eSbo5pNRFRL+mXpGsxlXRD0cbAOnmKfTRwiqQ/kEbmBpM+23ta2N5bkm4AfinphIh4Q9JawLeAByNiXjNVVOtMWi88E3hf0vbAIaSblipmkD6LxjwNvAJcKOl/gbVIa1GvaWY6HUmH5n7/A5gNLCKtDa60u72kThFRCZ9jSNf6N8CfSTdYbVjryZqZmdlHtfebhg4EPkUa3XoUuK5SEBFXku4Ov4Z01/NU0rRp8aaWR0mhcwZwN/AL8vrNfGfzmcCNkmZLOr2ZvjwMPAhMyvvvnEfO9gSOBd4hBZUzI+KWT3LSjTgduAX4LTAXmABUbpAaQQq/D5BuktoV6FMIxi1xBPA3YIKkuaTgtR/p5qKlEhEvAT8B7iQFvlP5cG1pxbnAwZJmSRrfQB2LgL1I091TSQH0KVIYbM6upDW884EnSD8DY3LZVaTlCu/kz3Ul0s/ZJcA40vVcH/hDzSdsZmZmH1FXX7/U+cHMGlF3wSL/g7IOo/6kdjsJZmbtR02PDWzvI5xmZmZmtpzzr6+ZpB2Bj03lZudFxHlLWd9ppIexN2SPiHh0aeprTwoPv6/2aETssUw7Q+t/dmZmZta6PKVu1orGjRtX369fv7buhpmZ2bLiKXUzMzMza3sOnGZmZmZWKgdOMzMzMyuVA6eZmZmZlcqB08zMzMxK5cBpZmZmZqVy4DQzMzOzUvk5nGatyH/a0sB/EtLMVih+DqeZmZmZtT0HTjMzMzMrlQOnmZmZmZXKgdPMzMzMSuXAmUmaJ2mHtu7HiuKTXm9JV0ka3YpdaqqtVyQNXBZtmZmZdUS+lTKLiM5Ls7+kemDHiHispC61SA5hiyJiUFv3pSlLe73LIGkC8FBEnNvWfTEzM+vIPMJZIkmrtGJddZKW+18QWvOamJmZ2fJhuQswkiYDVwC7AdsBk4H/iYjHcyA7DRgIrA08CxwfEX+pod4PRizz9OkZwMXAKcAawC3ADyJisaTn82EPSFoC3BQRg3Lffg18C/gqMAi4SdLRwAnAhsBLwMkR8WiNfToBOAT4HPAtSWsA5wGfBRYBvwOOi4i3JJ0CHJSP3T9Xs2bu83eAM4HNgOnAuRFxfQ19qJxTH+BLwETg6Ij4Y2GfI4Djge7Aa8DgiHgglw0BdiJ9Fofk73tUjxBL2hc4C+hJ+kyHRMRvCm0cDpwOdAPuJD33a1EN/d8dGJHPeyHwp4jYXdKlwI7ADpJOBd6IiC1zIB4OHAwsAUY214aZmZk1bXkd4TwcOA5YE3gQuDZvPxkYAOxJCnePAg9K6tqCNnoAG5CCyleB/YD9ASJim7xPn4joXDV9fQRwItAFuFPSAcA5uV/rAlcC90nqUWM/vg/0BzoDzwH/Bo4lBa8vABsBv8j9Oh+4Hrg296tzDpv/AVxNCq/rAIcCl0raqcY+HEUKlOsAtwH3Vq5pDpuDSUF3bVIovEPS5oXjdyKF3O7AvtWVS/p67veppGt0GnCjpO1y+Y7AZbkf65A+8/419v060i8OawIbA+cCRMSxpJ+Pc/J12jLvfyqwF/B1oBcpANf6WZmZmVkDlrsRzmxURLwI6eYR4ARJawKHAcMjYmIuG0oaZewL3LiUbbwHnBURi4FXJP0OECkYNeXKiHiuUoekw3J/n8rbrpY0CDgQGFZDPy6IiFfz68VAcc3oDEnnk0Ygm3I88IvCqOrTksaSQvAfaujD1RHxDICk4cAPSKHshlz30IiojPreK+lhUjivrI2cEhEX5tcLG6h/IHB7RIzP7++R9BvSLxZP5X7eFhEP5vLrJB1ZQ78r7W0GbBARM4AJzew/APhZRLySz/ckUug3MzOzFlpeA+f0wuv5+XsX0gjapEpBRCzJU8LdW9DGWzlsFtvpUsNxk6vedydNxxe9uhR9+kh9kr5CmlLfBlidNLXc3A04vUjT8ScWtq1EGuFbqj5ERL2kqcAmhbovk3RxYf+VgdcL76c0U3934Jmqba8CX86vNwGiqnwStdmbNGL6Z0kzgSsi4qIm9t+Ej57vfElv1diWmZmZNWB5DZyNmUaaAgVAUqf8floJbTX2N7OXNNWnrDcwrsZ2quu7iTStvV9E/FPSXlV1Ve8PKfCNjogRNbZZrWflhaQ6YFM+DJRTgJ9ExK1NHN9Qn4oau0aVz+2NBsp7Aq80Uy955LV/7vc3SetuX4iI3zfSr4+0ldfMdmuuHTMzM2tcRwuco4FTJP2BNEo1mHSO95TQ1gxgCz46xd1Yn34h6S4+vHHmS8ABLWy3KzAHmCtpU9Kaw+p+bS+pU0RUAtVFwGhJTwKPk0Y3vwDURUT1yGFDDs9T3H8GfkQaWa1c05HAEEl/B54HVgO+ArxdWdpQg2uBhySNAR4i3aC0D7BLLh9DWvc6GniENF2/Hc0ETkmrkq7zPRHxtqRZpJBZGbmeAWxeddgY4OT8yKR/AOez/K51NjMzaxc62v9IR5DWaj4AvAnsSrqx558ltHU6MFTSLEmjGtspIm4AzgbGAu8ARwN7RkRz08yN+R/SutS5wB1A9cjiVaS76t+RNFvSSvmO8SNI1+dt0pKEkTQ/FV9xBenGm1mkm3X6RsScfH5XkkLZNbl8Kulu+JoffxQR/0e6kemCXMf5wMER8WQufwT4YT63d4H/Am6usfr+wERJ84C7SKOxj+SykYDydXoxbxsG3A88SZq2n0rzSwLMzMysCXX19Y3NDJt98FikMyJibFv3ZXlQd8Ei/4My6k/qaJNHZmaNqqtlp442wmlmZmZm7cwK82t4njJt6HmKUyLic8u6PwCSxpMePv4xy+pPP0q6nPSQ84ZsvSz68ElIOghobEnDkbU83N7MzMzK5Sl1s1bkKXUDT6mb2Qqlpil1/1fRrBXdteV4+vXr19bdMDMza1e8htPMzMzMSuXAaWZmZmalcuA0MzMzs1I5cJqZmZlZqRw4zczMzKxUDpxmZmZmVioHTjMzMzMrlR/8btaK/OD39sUPYDczK53/lrqZmZmZtT0HTjMzMzMrlQOnmZmZmZXKgdPMzMzMSuXAaWZmZmalcuC0D0gaL+mUtu5HeyPpKkmj27ofZmZmyys/M8Q+EBF7tHUfliVJQ4BvRsTubd0XMzOzjswjnGZmZmZWKo9w2gckTQAeAsYCk4ABwI+B7sATwKERMT3v2xkYAuwDdAOmAUdGxKOSVgeG5bJPA48Bx0XE1EI7zwK9gN2Bt4D/IT089iJgU+B3wICImJuPWRc4H+gDrAY8DPwwIt5s5px6AqOA7YD6fF4HAF8CTgM6SZqXd/9iRLwm6XDg9Hxed+Z+LVqKS2lmZmYFHuG0pvQHdgI2BtYAhhbKriaFuN2ArsC3gem5bCSwff7qAbwNjJO0UuH4Q4CfAWsBNwNjSKFzJ6AnsCVwHICkOuC3pMD4+VznXOCGGs7hPGAqsAGwHjAQmBURN+eyCRHROX+9JmlH4DLgKGAd4MF8HczMzKyFPMJpTTk7It4GkHQDMCi/Xh/4b+DzETEp7/tKLusEHAr0i4g38rYTgHeBr5FGSgFuiYincvlY0kjqiIh4N2+7G1De9yv5a/eI+HcuPwV4W9ImEfF6E+ewENgQ6B0RLwEvNHPOA4DbIuLB/P46SUc2c4yZmZk1wSOc1pTphdfzgS75dc/8/eUGjukGfIo0dQ1ARMwjTZt3b6TuBY1sq7TXK9f5pqTZkmYDrwL/Ik2/N+Xk3JdxkqZLuiQvB2jMJsDkqm2TGtjPzMzMauQRTmuJyfn7FsBfq8pmAv8mhdLKqGdnYH3SOs+WmEIKvOtExJKlOTAiZpKm5o+T1Ju0JvMU4Cygobre4MNAXdGTfC5mZma29Bw4balFxFuSbgN+KWkgKRBulstekXQdcI6kvwKzgQuBicDTLW0SeB64WNJPIuIdSd2A3SLipqYOlNQ/tzsZmEOaYl+ci2cAm0paNSIW5m1jgPvyczcfAfYnrVV14DQzM2shT6lbSx0O/IkUyuaSRg43zGU/IoXEP5Ju2PkM8O2IWNxAPc3Ko5p7k+4Wf0bSXOBJYJcaDt8293Ee8CLp7vgRuexW0qjrjDxV3ysiHgF+CFxFWnf6X6SbmszMzKyF6urr69u6D2YdRt0Fi/wPqh2pP8mTOGZmJaurZSePcJqZmZlZqfzrvy3XJG3Kx29cqhgbEUcty/6YmZnZx3lK3awVjRs3rr5fv35t3Q0zM7NlxVPqZmZmZtb2HDjNzMzMrFQOnGZmZmZWKgdOMzMzMyuVA6eZmZmZlcqB08zMzMxK5cBpZmZmZqXyczjNWpH/tGXr8J+kNDNbbvg5nGZmZmbW9hw4zczMzKxUDpxmZmZmVioHTjMzMzMrlQOnmZmZmZXKgdPahKRdJC1q637UStIiSbu0dT/MzMyWRw6cKxBJAyW90tb9aE8kTZZ0cFv3w8zMrCNz4FzGJK2yIrVrZmZm5qcrtwJJk4FfA32ALwETgaMj4o+SRgOrAO8D3wZuBo6WtDNwPrAVMB0YGRGjcn27AA8B3weGAl2Au4BjI2Je3mfdfHwfYDXgYeCHEfFmVZ++BXwV+CnwE2BVSfNy1/cChgM3RcTIwvkMBb4REbs1cc4jgC0i4juFbbsA44ANgXpgLPB1YHXgFWBwRDzYSH2jgUURMajqup4REWPz+x2BYcDWwCzgl8DPI6LJh61L2j+f+ybAAuC+iDhU0jhgU+AqSZcDj0dEH0ldgEuBfsBc4Kym6jczM7OmeYSz9RwFHA+sA9wG3Cupay7bDxgPdAP+V1Iv4D7gV8C6wEBgmKT9CvWtRAo8XwT+H/BZ4OcAkuqA35JC3eeBHqRgdENVn44ATiQF1otyH1+LiM75awIwihRsyXV3Ag4DrmzmfK8B9pTUrbDtMOCWiJhP+tm6A9gin+ONwO1V+9dM0tbAvcAI0nXsCxwLHNLMcasDY4BjIqIL0Bu4CiAi+gFTgUH5evTJh12U+7016frvTfo8zMzMrAU8wtl6ro6IZwAkDQd+QBpBBHgsIm7OrxdIOgB4NiJG521PShoFDAJuLdQ5OCLmAHMknQXcLeko4MvAV4DdI+Lfuc1TgLclbRIRr+fjr4yI5/Lr9yQ11O+bgJGSto+IJ4H/BD4N/Kapk42Iv0p6Djg4H98F+B5pxJU8Eju2cMgISYNJo633NlV3I34A3BoRd+b3EyVdCgwArmvm2PeBrST9KSLeBR5tbMccuA8C+kbEjLxtMPDdFvTZzMzMcOBsTZMrLyKiXtJU0hTuR8qy7sCkqm2vkkbSiqZU1f8pYD2gV379ZlWI/Bdpivj1wjFNiogFksaSwu6T+ft1lSDbjGuAo4GRwH8Dr0fE/wFI+jRpNHLP3OclpJHWFo1wks55V0n7FLZ1AqY1dVA+vz1JI70/lfQacGFEVI8GV3QjXdvJhW3Vn5WZmZktBQfO1tOz8iJPeVeC39aksFU0jRTEinrz8fDUgxREK/X/G3ibFETnA+tERHXdRdVlje07Cvg/ST8lTeN/qYk6iyqjo18mLQu4plB2IrATsBswOYfwt4G6RuqaSwqmAEhaGVi/UD4F+HVEHFNj3z6Qlw5MkLQSaR3t7ZKeiohX+fg1eRtYSLrexWtvZmZmLeQ1nK3ncElfzneDn0y6UeaeRva9EfiKpAGSVpb0NeBI4Oqq/YZJ6ippfWAIMCYHzACeBy7ONw8hqVu+OaYpM4D1C2tLAYiIF4AXgduBpyPir7WccETMJk29nwtsD1xbKO5KCsjvkG5UOgtYq4nqngF2k9RL0qdINzkV76z/JbC/pH6SVsnXbet881WjJG0gaV9Ja0bEYmB2Llqcv88grdesnNNi0lrYs/OxXYGfNdWGmZmZNc2Bs/VcAVxMunu6P2kN4JyGdoyISaQRzmNJgWwMcGZE3FLYbTEpsP4Z+BvwGmnUkBw69yaNFj4jaS5pOnyXZvr4MPAgMEnS7KqwNgrYluZvFqp2DbAHcH9ETC9s/zkp3P2DNFK4gKan+K8n3Yn/bN5/KvBGpTAi/kJaE3sC6a7+t4DRND9F3wk4Bpicr9NlwKERUenLucDBkmZJGp+3HU+aRp9Iuv7j+DCgmpmZ2VKqq69v8okyVoPqx/e0Qn27AA9FxDJb8pDb/C2wUUQsWFbtdjR1FyzyP6hWUH+SV/uYmS0nGlsq9xEe4TQkrQacRLqr3WHTzMzMWpWHEVZw+a7vsaSp7J9WlY0HdmzouIjoXH7vaiPpNOC0Ror3iIhGH4NkZmZm5fOUulkrGjduXH2/fv3auhtmZmbLiqfUzczMzKztOXCamZmZWakcOM3MzMysVA6cZmZmZlYqB04zMzMzK5UDp5mZmZmVyoHTzMzMzErl53CataLW/NOW/vOOZma2HPBzOM3MzMys7TlwmpmZmVmpHDjNzMzMrFQOnGZmZmZWKgdOMzMzMyuVA6eZmZmZlapDBk5JEySd0db9aE8k1Uv6Zlv3Y2lIelFS/7buB4CkRZJ2aet+mJmZLY/8oL+SSBoInBERm7d1X9o7ST2BSUD3iHi9sj0iPrcM2p5M+pzGlt2WmZnZiqpDjnBa+yFplbbug5mZmbWtDj/CKemLwEXAtsAs4NfAsIhYnMuvAXYH1gKmAedGxA25bBfgIeAg4DxgPeB+4PsRMbeJNncALgdWlTQvb94rIiZI2hk4H9gKmA6MjIhRzZxDX+AaYOOIeD9v6wzMAPpGxCOSzgP2B9YH3gQuiYjAgtFAAAAgAElEQVSLGqlvIFWjr5JGA4siYlB+vynwc+CbQD0wDvjfps47HzcE2Al4Fjgkf9+jqesMPJ+//01SPTA8Is6pHn1sybXLx+0P/ATYBFgA3BcRh0oaB2wKXCXpcuDxiOgjqQtwKdAPmAuc1VwbZmZm1rgOPcIpaU3gQeBhYEOgL3A4cGJht8eAL5GC0FBgtKStC+UrAX2AbYDPkoLrcU21GxFPAEcBr0VE5/w1QVIv4D7gV8C6wEBgmKT9mjmV+4BFuf8V+5EC5x/y+7+SwmEX4Ihc7382U2+DJK0G/D7X2QvYmhTWflFjFTuRAmF3YN+8ranrvE3+vmW+Vuc00KcWXTtJqwNjgGMiogvQG7gKICL6AVOBQbndPvmwi4At8nl/Edib9HNgZmZmLdDRRzj7AgtJo2n1wEuShpMC5wiAiLi6sP9Nkk4CdiGFrYpTI2IeME/SbwG1sD8HAM9GxOj8/klJo4BBwK2NHRQRiyWNAQ4Dfps3HwZck8+LqjWIv5d0D7AbaUR2ae0F1EVEZWTvPUlnAo9LOqIyOtyEKRFxYX69MPevluvclBZdu+x9YCtJf4qId4FHG9tRUifSiHbfiJiRtw0GvltjP83MzKxKRw+c3Unhp76w7dW8vRIuhgD9SSOg9cAaQLfC/osjYmbh/XzSKGJL+zOpaturpBG05lwDvCBp/dz+14EDK4WSjiONbG4C1AGfBm5ooJ5a9AI2lTS7ans96Tq90czxU4pvarzOzWnRtYuIBZL2JP2S8VNJrwEXFqbzq3UDPgVMLmyrbtfMzMyWQoeeUietFewhqa6wrXfeDmnUbBBp2nftiFiLtJ6wjk9uSSP96Vm1rdifRkXEROAZ4GDSdPJDlTu6JX0DGA4cCayXz2McjZ/HXFLgK9qo8HoK8HJErFX1tVpENBc24ePn3tx1buhaVfsk125CRHybtAb3XGCspM0aaftt0qhssa3qds3MzGwpdPQRzntI6/FOkzSCNHI3GKjcaNKVtDZyJtAp30yzDXB3K7Q9A1hfUteI+GfediNwpqQBpNHHL5NC4tE11nkNaf1oF+DkwvauwGLSedTnm4z2oPGp5j/lvu0F3EsaJdwJqEzL300aDTwNuASYRwqkX4uI39TY16LmrvNMUvDbAni9oQpo4bWTtAFpbetDETGnMGpbWRYwI7cLfLB84QbgbEl/Ad4Dflb7qZqZmVm1Dj3CGRFzSDf87E66c/t+4DrS3dcA1wJPAa+Qpom3pon1fUvpYdINS5MkzZa0c0RMAvYEjgXeId3McmZE3FJjnTeRRvU6A3cWtlfO62nSCN33gEaDYUS8ChwPXAG8C/wXcHuhfAGwK+l6TATmAL8j3fTTEk1e54h4DzgTuDFfq9Mb6HNLr10n4BhgsqS5wGXAoRExOZefCxwsaZak8Xnb8aRp9InAn0mjxc2tWzUzM7NG1NXX1ze/l5nVpO6CRa32D6r+pI4+AWFmZh1ATcsQO/QIp5mZmZm1PQ+htJCkHYHxjRSfFxHnLWV9pwGnNVK8R0S01lT/J9La590a2tO1u2vL8fTr129ZNWdmZrZc8JS6WSsaN25cvQOnmZmtQDylbmZmZmZtz4HTzMzMzErlwGlmZmZmpXLgNDMzM7NSOXCamZmZWakcOM3MzMysVA6cZmZmZlYqP4fTrBXV+qct/Wcrzcysg/BzOM3MzMys7TlwmpmZmVmpHDjNzMzMrFQOnGZmZmZWKt+5YMuMpFOBHwFrAN+KiD8uo3YvBxZFxLHLoj0zMzP7KN+lbsuEpE2AqcDnI+Kvbd2fsvgudTMzW8H4LnVrV3oCSz5J2JS0Sut1x8zMzJYVj3DaR0jqDAwB9gG6AdOAI4FngGF5+6eBx4DjImJqPm5C3qcn0Ad4CzgxIu6U1B8YDawGzAfejIjNJK0LjMz7A9wP/Cgi3s11TgZ+DXwL+CowCNgK2BEI4HDSL00/BW4Hrsn7vQwcHBEv5XpGk6bUB+X39cAxwGG5vheBgRExMZd3AS4D9gLmAmfmfuweEROaun4e4TQzsxWMRzitRa4GtgN2A7oC3wamk4Lh9vmrB/A2ME7SSoVjDwUuBNYELgWulbR6RNwM7AEsjojOEbFZ3v96YG3g/+Wv9YAxVf05AjgR6ALcmbftBPwd2BA4GBiR+30MsA7wEnBxM+c5ENg3tzkNuKRQ9gugNymMfgHoC6yEmZmZtYiHWewDktYH/pu0znJS3vyKpE6kMNkvIt7I+54AvAt8DXgi73tzRDyey68Afg5sATzfQFsbAf8JfDYiZuVtJwITJX0mIqbnXa+MiOfy6/ckAbwcEVflbeMlvQPcXxjRvIEUZpsyojA6OxoYm1+vBBwE7BERb+Vtp+XrYmZmZi3gEU4r6pm/v1y1vRvwKaASQomIeaRp8+6F/aYXyufnl10aaaty3KTCtlerygAmN3Ds9Kr3C6q2LWii3YbqmF/Yfz1gVWBKobz42szMzJaSA6cVTc7ft6jaPhP4Nx8G0spaz/VJ09EtUTmuZ2Fb76oygCUtrL+l3gYWkpYNVGy6jPtgZmbWoXhK3T4QEW9Jug34paSBpJG9ynrL64BzJP0VmE1aqzkReLqFbf1D0gPAhZIOJS06vhAYX5hOX+YiYnGekh8i6c/Av0g3JZmZmVkLeYTTqh0O/Al4hHSH9p2km3N+RLoz/I+k52l+Bvh2RCz+BG0dnNv4Gym8zgYGfIL6WsvxpHN8GfgL8CBQTxrlNTMzs6XkxyKZNUPSlqRAvHFE/KOpff1YJDMzW8HU9Fgk/1/PrIqk3qRR3adINxGNBP7QXNg0MzOzhnlK3ezjVgOuAOYAfybd9X5gm/bIzMxsOeYRTrMq+c9vfr6t+2FmZtZROHCataK7thxPv3792robZmZm7Yqn1M3MzMysVA6cZmZmZlYqB04zMzMzK5UDp5mZmZmVyoHTzMzMzErlwGlmZmZmpXLgNDMzM7NS+W+pm7Wipv6Wuv9+upmZdUA1/S11j3CamZmZWakcOM3MzMysVA6cZmZmZlYqB04zMzMzK5UDp5mZmZmVyoHTzMzMzErl57RYi0maADwUEee2dV8qJP0UOBBYF/gX8AfgxIiYWthnM+ACYNe86SVgx4h4P5cL+CXweWA68JOIGLvMTsLMzKyD8QintRlJq5RQ7RjgSxHRFegJTAVuKrTZDXgUeB7YFFgHOBZYnMvXBMYDtwNrA0cBl0vaoYS+mpmZrRA8wmktIulSYEdgB0mnAm9ExJaSjgCOB7oDrwGDI+KBfMwQYCfgWeAQ4FlJRwOTgIHAYKAH8AhwUH5/OLAEOCciLmuuXxExsfC2Lh+7ZWHbicDUiBhSPKzweh9gAXB+RNQDD0r6DfA/wBPNtW9mZmYf5xFOa5GIOJY0UnhORHQuhM3BpLC4NnA6cIekzQuH7kSapu4O7FvYvi/wTdKoY0/gKeBVYCPgMOAiSZvW0jdJB0qaA8wjhd8hheJvAdMk3SPpXUkvSDqoUL4N8FwOmxXP5u1mZmbWAh7htNZ0PDA0Ip7P7++V9DCwP1BZ5zklIi7Mrxem5ZJACq7vAki6G+gbEVfmsvGSZgHbkqbImxQRNwA3SNoQ+D7w50LxesBXgf7A3qQAOk7SlIh4DOgCzKmqcjbQtdmzNzMzswY5cFpr6gVcJuniwraVgdcL76c0cuz0wusFVe8r27osTWciYoakK4HXJG2aA+1c4ImIuC3v9qCk+4BvA4/l8p5VVa0F/HNp2jYzM7MPOXDaJ7Gk6v0U0h3dty7FMWVbGViDNDX/LvAnYPMG9qtMoT8PfKeqbNu83czMzFrAgdM+iRl8NLyNBIZI+jspoK0GfAV4u+pmnlJI6gT8ALglIt6StAlwCTAZqLQ/CnhU0neAu4CdgT7A8Fz+G+B8SScDvyDdGLUP8B9l99/MzKyj8k1D9kmMJD22crakF/Oay/OBa4BZpPWWZwJlPP6oMXsCf5E0n3Tj0QJg94hYBBART5Ke0zmcNH1+CXBoRDyRy2fnOvYjreW8EjiqUm5mZmZLr66+vr75vcysJnUXLGr0H1T9SZ5QMDOzDqeulp08wmlmZmZmpfKQiy1XJF0OHNxI8dbFP2FpZmZm7YOn1M1a0bhx4+r79evX1t0wMzNbVjylbmZmZmZtz4HTzMzMzErlwGlmZmZmpXLgNDMzM7NSOXCamZmZWakcOM3MzMysVA6cZmZmZlYqB04zMzMzK5UDp5mZmZmVyoHTzMzMzErlwGlmZmZmpXLgNDMzM7NSOXCamZmZWakcONuQpAmSzmjrfljzJL0iaWBb98PMzGx55MBpyx1Ju0ha1Ep1OfSbmZmVzIHTzMzMzEq1clt3wBJJXwQuArYFZgG/BoZFxOJcfg2wO7AWMA04NyJuyGW7AA8BBwHnAesB9wPfj4i5zbS7DXAx8DlgJeBJ4NiIeDWXj87b3wf2AeYDJwEvAVcCWwEBHBQR/8jHrAuMBPrkZu4HfhQR7+byycAZETE2v+8JTAK6R8TrhTb/BeyX2xwaEaMkbQSMB1aSNC/Xf0xEXNvEOe4OjAA2AxYCf4qI3SVdCuwI7CDpVOCNiNhS0irAcOBgYEk+FzMzM2shj3C2A5LWBB4EHgY2BPoChwMnFnZ7DPgSKXAOBUZL2rpQvhIp4G0DfJYUXI+rofl6YAiwMdATmAeMrdrne8DtwDrAOaSgORT4LrBBruPswv7XA2sD/y9/rQeMqaEv1W2Oy23+ELhUUo8cavcAFkdE5/zVaNjMriOF6jXzeZ4LEBHHAo8C5+R6tsz7nwrsBXwd6EW6Lj2Wsv9mZmaWeYSzfehLGnk7NyLqgZckDScFzhEAEXF1Yf+bJJ0E7AL8tbD91IiYB8yT9FtAzTUcES8U3v5b0tnAnyWtHhEL8vbfR8Q9AJKuA34FjImI1/O224Aj8uuNgP8EPhsRs/K2E4GJkj4TEdNrvCa/j4i78us7JM0mBe4pNR5ftJA0urlBRMwAJjSz/wDgZxHxCkC+1t9vQbtmZmaGA2d70R2YksNmxat5O5I6kUYh+5NGQOuBNYBuhf0XR8TMwvv5QJfmGpa0GSnUbpf3r/ShGx+Guw9CYkQskPSRbcCCQlvd8/dJVedSKas1cFbvV9P5NGJv4DRSkJ4JXBERFzWx/ybA5MqbiJgv6a0Wtm1mZrbC85R6+zAN6CGprrCtd94OcAAwCNgXWDsi1gKeB+r45C4H5gJfjIiuwDfy9pbWXelzz8K23lVlc0mBuWKjpWxjydLsHBHPR0R/YH3gSGCYpF2bqOsNCv2XVB3uzczMbCk4cLYP9wCfAk6TtKqkLYHBQGUavSuwCJgJdJJ0OGmtZmvoSho9nC1pPdLazBbLaywfAC6UtJaktYELgfGF6fRngAMkdZbUDThzKZuZQbppqFdzO+breaik9fII8ixSyFxcqGvzqsPGACdL2kzSp4Hz8b8VMzOzFvP/RNuBiJhDuuFnd+BN0l3d1wE/z7tcCzwFvEIafduadLNLa/gR6U7tf+Y6726FOg8mjWL+DZgIzCati6w4gxT4ppPWU960NJVHxMukdaRPS5ot6ZBmDulPWkM6D7gL+ElEPJLLRgLK9byYtw0jfQZPkpYGTKVla0fNzMwMqKuvr29+LzOrybhx4+r79evX1t0wMzNbVmpagucRTjMzMzMrle9S7+Ak7Uh6UHpDzouI85Zlf8og6SBgVCPFR0bE9cuyP2ZmZvZRnlI3a0WeUjczsxWMp9TNzMzMrO05cJqZmZlZqRw4zczMzKxUDpxmZmZmVioHTjMzMzMrlQOnmZmZmZXKgdPMzMzMSuXAaWZmZmalcuA0MzMzs1I5cJqZmZlZqRw4zczMzKxUDpxmZmZmVqoWBU5Jm0iql9Szhn2HSHqoJe0sC5Iul3RpW/ejPWvLa9QabUs6SNLzrdUnMzMzWzort3UHcmidBHSPiNdr2H8gcEZEbN4a7UfEUa1RT0chaTLp+o6tbGvLa9QabUfE9cD1rdAdMzMzawFPqa8gJK3S1n0wMzOzFVNNI5ySNgSuAHYG3gTOryo/Ajge6A68BgyOiAcKu9RJGgkMAN4DLo2In+WyylTn3yTVA8Mj4pxG+rEDcDmwqqR5efNeETFB0s65X1sB04GRETGqhnMbDSyKiEH5fT1wDHBYrutFYGBETMzlqwAnA4cCGwFv5fO9TdLKwGnAQGBt4Fng+Ij4S6GtlYD3gX2A+cBJwEvAlbm9AA6KiH/kY1YHhgL7AmsCTwPHRsQrzZzXLsBD+TzOBroBXSQdDxwNbAzMIo38nRERiyWNAzYFrpJ0OfB4RPRp4Br1AC4GvkH6PG8HfhwR7zXTpyHAjvkcDyf9wvPTfPw1wFeBl4GDI+KlwjVbFBGDJNUB5+Zz6gK8A1wYEZdIWpv0M7or6ef6deCoiHi0elRc0gTgGaAn0If0GZ4YEXfm8jrgx8APgNWBa4EvAo9GxJCmztHMzMw+rtYRzuuBxaQwshMpUAEfhM3BwEGkkHU6cIek4pT3TqSg+hlgb+BESQfmsm3y9y0jonNjYRMgIp4AjgJey/t2zmGzF3Af8Ctg3dy/YZL2q/H8qg0kBbz1gGnAJYWyc4GDgf2ArqQQ/nIuO5kUqvcENgQeBR6U1LVw/PdIAWsd4BxS0BwKfBfYAKgnBcSKShDdPtf5FHB3jSOWK+W+bJvrhhTE9sh935sU/AYBREQ/YCowKF/bPtUV5lB9DzAD6JH79Q3gghr6A+ln4e/5XA4GRgBXk0L+OqTwfXEjx/4HKehvFxFdgK8Bj+Wyk0nhsAewFul6NrVE41DgQlKIvxS4Nod7gENIv0D1I1236bnfZmZm1gLNjnBK2pg0arR5RMwB5kg6G6iMYB4PDI2IykjlvZIeBvYnhTNI/8MeHhH1wDOSriCFuhta6TwOAJ6NiNH5/ZOSRpGC1K0tqG9EREyFD0bYxubXdaRg1D8iXsj7vs6HweYw0nlWRkOH5j70BW7M+/w+Iu7J5deRQvKYyvpVSbcBR+TX6wEHAj0i4s287WzgBGA7PgxbTRmcPzcAIuL2QtlzksYAuwHNjgZnXwO2IIW++cB8SWcAv5V0bP6Mm/JyRFyVX4+X9A5wf2FE8wYaX2+5EFgN+JykmRHxFml0slK2LrAl8FxEvNxIHRU3R8Tjuc0rgJ/n83qe9EvDqIh4LpePIH3uZmZm1gK1jHBukr9PKWybVHjdC7hM0uzKF/At0pRtxZSqIDK5UG9r6F7VJ4BX8/aWmF54PZ80fQtpWnoNPhzRbLIfEbGEdK7FfkwvlC9ooL0FhfZ65e8vFK7tu8Aq1HZuS0gjtB+QdICkP0p6R9IcUpDqVkNdFd2BmTlsVrxKCoK11DO96v0CGj//j4iICaQlC2cAb0l6QJJy8Qjgd6Tp75mSrpW0QUP1VPejcC6Vdjem8POef3Y/ch3NzMysdrWs4Xwjf+9BChaQ1r5VTAF+EhFNjST2kFRXCJ09+XBUcEltXf1AQ/tPI00dF/Wm9UPCTFIg2oI0LdxQP3pW3kjqlN+3tB+V0LNFRMxswfH1xaAvqTtptHYfYHxELJR0AaDCMc19HtOAbpJWLwTm3sC/SNenVBFxBXBFnv4eAtwBbJpD4+nA6XnN8VhSCB3QgmbeIP28Ax+MbLf0lxczM7MVXrOBMyJezzdZnC/pMODTwFn/v707D7ejKPM4/n3JAoQQQEkUCSGJgMsAKil4cGSJKJuCooIIAiKKiSMiMqJOIJoRZBNk5BkyrCYsDoiMhgkgYREelucRpgSVPRASAoGYAElIBIFAzR9vHdK3c865596c5Yb7+zzPee7tvbq6uvs91VWnC7OcA0wJITyOP45cDxgHPF95tIy33TwhdxzaFn9kfHyethgPcramfpu7ioXAiBDCsBjjS3nclcDkEMIR+GP6HYAJeOeYpokxphDCVDwv5uMdijYH3pEfsU8Hvh9CuAOv2fwBnsfX93J7i/Ij5qkhhONijAtCCBvjNcg3xxhXdLOKsqF4rfZi4PUQws54e8VHCvMsxI9FLfcCTwBnhxD+FW8veTIwrYHH6WskhLATsG5Ow6vAcrxtMSGE/XO6ZgMr8AD4jV5u6nLgjBDCb/FjfCzeQUxERER6odFOQ4fiN/qn8Y4wl1UmxBgvwnuHT8N7Pc8HJuOPfSvuxIPOhcB1wC/I7Tdzz+bJwJX5sfGJ3aTlNuBmYG6ef/cY41y8hvMYvOfy5cDkGOPVDe5fT5wIXA3MwAOe24FKB6mf4cHvTXgnqT2AvQqBcW8cDTwG3B5CWA48gHdY6nFwl9tJ/hi4FlgK/JBVbUsrTgEOCyEsCSH8vso6VgL74U0i5uPB3z14b/tWG4qXnefx47wXcHCe9l5gJvASHuy/ggf8vXEZcB5wA34cRwJ/xINcERER6SFLqaWVUiJrvdw04mnghBhj3Y5uM2fOTPvvv397EiYiItJ51shMHX/TkEhfFEL4El6LvQ7+m5xDgNVqfEVERKR7fS7gDCHsSu0b+6kxxlN7uL5JeM/mavaNMd7Zk/X1JYUfvy+7M8a4b1sTQ/OPXYcdw6qfinoQ+FSMcUkH0yMiIrLW0iN1kSbSI3UREelnGnqkrnepi4iIiEhLKeAUERERkZZSwCkiIiIiLaWAU0RERERaSgGniIiIiLSUAk4RERERaSkFnCIiIiLSUgo4RURERKSlFHCKiIiISEsp4BQRERGRllLAKSIiIiItpYBTRERERFpKAaeIiIiItJQCThERERFpKQWcIiIiItJSCjhFREREpKUUcIqIiIhISyngFBEREZGWspRSp9Mg8rax7rrrPvjaa6/9o9Pp6IsGDhy46cqVK5/vdDr6KuVPfcqf2pQ39Sl/amtS3jyfUtqn222t4UZEpGC77bb7R4wxdDodfVEIISpvalP+1Kf8qU15U5/yp7Z25o0eqYuIiIhISyngFBEREZGWUsAp0lwXdjoBfZjypj7lT33Kn9qUN/Upf2prW96o05CIiIiItJRqOEVERESkpRRwioiIiEhL6WeRRHoohLANcCnwTuAF4IgY4+OleQYA5wL7AAk4PcZ4cbvT2gkN5s9k4EvAG8DrwKQY46x2p7XdGsmbwrzvA+4HpsYYv9e+VHZOo/kTQvgiMBkw/Pz6ZIzxb+1Ma7s1eF6NAKYBWwCDgNuAY2OMK9uc3LYKIZwFfAEYDWwXY3ywyjz9+ZrcSP60/JqsGk6RnjsfOC/GuA1wHnBBlXm+DGwFbA18FJgSQhjdthR2ViP5cy+wY4xxe+Ao4NchhPXbmMZOaSRvKjfHC4AZbUxbX9Bt/oQQAjAF2DPGuC2wC7CsnYnskEbKziTgkXxebQ+MAz7fviR2zAxgN+CpOvP052tyI/nT8muyAk6RHsg1CDsAV+ZRVwI7hBCGl2Y9GLgoxvhmjHExfsIf1L6Udkaj+RNjnBVjfDkP/hWvqXpn2xLaAT0oOwA/BK4DZrcpeR3Xg/z5LnBWjHEhQIxxWYzxbf12rx7kTQI2DCGsA6wLDAYWtC2hHRJjvCvG+HQ3s/XLazI0lj/tuCYr4BTpmS2ABTHGNwDy32fz+KJRdP02Ob/KPG9HjeZP0RHAnBjjM21IXyc1lDchhA8BewPntD2FndVo2fkgMDaEcEcI4b4QwkkhBGtzWtut0bw5GdgGeA5YCMyKMd7dzoT2Yf31mtwbLbkmK+AUkY4JIeyO3yQP6XRa+oIQwiD8d/EmVoILWc0A/HHxnsDuwL7A4R1NUd9xEF47tRmwObBbCOHAziZJ1iatvCYr4BTpmaeBzXMbu0pbu/fk8UXzgS0Lw6OqzPN21Gj+EEL4KHAFcECM8bG2prIzGsmbzYD3AjeEEOYBxwFHhxD6ww9X9+TcuibG+GqMcTlwLbBTW1Pafo3mzbeBX+XHxsvwvPl4W1Pad/XXa3LDWn1NVsAp0gMxxkXAn1n17e8Q4P7cJqjoN3igsE5uZ3UAcE37UtoZjeZPCGFH4NfAgTHG+9qbys5oJG9ijPNjjJvGGEfHGEcD/4G3O/tG2xPcZj04t/4b2CuEYLlG+BPAX9qX0vbrQd7MxXthE0IYDHwSWK1Hcj/VL6/JjWrHNVkBp0jPTQS+HUKYjdcoTAQIIdyQe9ACXA48CTwO/BH4SYxxbicS2wGN5M9UYH3gghDCn/Nnu84kt60ayZv+rJH8uQpYBDyMB2EPAZd0IK3t1kjeHAfsGkJ4AM+b2cBFnUhsO4UQzg0hPAOMBG4JITyUx+uaTMP50/Jrsl5tKSIiIiItpRpOEREREWkpBZwiIiIi0lIKOEVERESkpRRwioiIiEhLKeAUERERkZZSwCnSj5nZ3mZ2Z2F4vJnN62CS2sbMppvZxU1c32gzS4Xh4Wb2lJlt2sCyE83s8malZW1gZrua2dJOp6M/MrPDenKeN/tckfpadW704rifbmYnN2v7CjhF+ikzM/x93T/uZr5vmtmDZvaSmS0xs2hmBxemzzOzw6ost9p4c7PzuoaWpo03s2RmK/LnWTObZmbvWLM97YyU0mL8R8q7y98NgJ8AU9qQrD4jpXRnSmnjTqejFjObYma3dDod/UGr8trMbjezk5q93lYrnxsdLItnAN8ys82bsTIFnCL9117AYOC2WjOY2SF4wPQ1YCP8dXrfBZb0cpsfB8YCb1L9Xb1vpJSGppSGArsAH8XftrO2+iXwVTMbVmeew4AHUkpz2pSmLsxsgJnpXiAiXaSUlgC/ByY0Y326yIi0Qa7tO8nMbsu1dw+Y2fZmdoiZPWFmy8zsYjMbWFhmlJldY2YLzew5M7vQzDYsTD/VzJ7M65tjZscVpo3OtYWHm9nDZrbczG4ys80KyToAuCXVf/vDPwN3pJTuSe6V/O37pl5mxQTgRvytH3UvYimlJ4HrgI+Up5nZwJwnB5TGTzezafn/T5jZPblWdrGZXWVmI2ptL+fXLoXh8Wa2sji5G78AAAwFSURBVLTNSbmGdqmZ3W1mdd8OlFJ6HHgef8VgLQcAN5fS8h0zezQft/lmdpqZDcjTfmZmM0rzj8/zbpCHtzWzWXm/K8sPytMqZeNrZvYw8DIwwsy+ZGZ/ybXPz5nZBZX15eXebWYzc1mdnZdPZja6MM/RuTZ8mZndb2Z71drpKvk73cwuN7Nf5vxdkM+PD5vZ/+X9u83M3lNYZp6Z/cjM7srnQTSzHQvT65YBMxuUj+ljef1zzOxA8xr8ScB4W1XjPrbGfuyet7EsH7MJhWnjzWylmR2c173MzK4unsdV1teba8X2ZvaHvJ9P5uUHFKbvlPNmhZndhX/pK25ziJmdZWZzzexFM7vRzLaqlcYqaX6nmV1mfq1aaGaXWuHJhJWedhTK4MhaeW1mR+b9/UEuj4vM7Owq5XhkYb1HmtkT+f//BHYFJud1Vn03uHnt4a1mdkYuIy+Y2fFmtmXO0+Vm9icz+0BhmTU6Vwpl/aJCWV+t3OT/6+ZPaV+6NH1o0nG/Gb9GrbmUkj766NPiDzAPf6XaB4BBwBXAHOBCYANgFP66vi/n+dcDnsAfta4PbALcAPyysM7D8BpHA/YAXgH2ztNGAwkP2DYFhgF3AxcVlr8HOLaUzvHAvMLwQcA/gFPwd1ZvXGPfDutuPDAceBX4PB5EJmBcadsrC8NbAY8V97m0/jOBGYXhocAKYNc8vAuwIzAQeDdwB3BlYf7pwMWF4QTsUic9P815NhYYgNf6Pg9sUszzKumcCZxSp2z8DfhMadwXgDH52H4kzzMhT/sg8BowvDD/pcAl+f8RwAt4QD8Y2ByIwI9KZePWnC+D8/7sC/wTXhGxFf7qyNMK27gV+J9clkYAt+f1jM7Tj8bL7IfyOj6Vj8dWNfa7nL/T8TL86bz8xLz8/+Kv5BsC/IGuZXge8CwwLu/HD4HFwLAGy8AZeT+3z3k9Etg+T5uCfyGrd16PyWk+Mm9jZ+BF4KDCPib81ZtDgXfh14ETm3it2CiXj8nAunm5J4ETCtNfyHkzOOfHQrqe57/CrxXvyvP8O/AoMKjauVIlzTfi5XyT/LkeuL7OtWB0zpeRtfI65+nrwHn4NfC9+Ks6J1VbR2GZJwrDtwMndXMMp+TtfJ1V58EbwC2lY3BzYZk1PVem4+XmM3kdn89p2LLGuVErf54ojXvrODXjuOd5xuFPpAbXy8dGPm296eqjT3/95AvuCYXhT+ULUDFouBo4J/9/IDCntI5xeMA2oMY2rgHOzP9XLsY7FqZ/C7i/MDwbOLK0jvHFC1Ietx/wW/ym9gb+CH7b0r79HVha+rxJ15vM9/EbZeUmdh9wQWnbKS+7BJgLnE+VIDfP/wE88BqRh48CZtc5BvsBiwrDb12c83DNgBMPRpYDu5XW+UBlH6kdcP4KmFonXa8B47spP2cBVxeG7wG+m//fMOf/x/Lw94A/lJb/AvnmVCgbu3WzzWOAe/P/I/MyYwvTP0HXm+iDwBGldcykxg2f6gFnMUgZktd/UGHcv9C1DM8DTi4MGzAfOLS7MpDnXQF8usa8U+g+4JwE3F0adxowq1Smi+f5z4Df1VnnPHp2rTgUeJr8quo8bgLwWP7/yzlPitN/Sj7P8S+kCRhVmL4OsIx8PlAn4MS/9CZg68K49+VxmxX2qTcB56vAkMK4r5PP8fI6Csv0JuB8qDRuUZVjsKSJ58p0CmU9j1sMfLbGuVErf+oFnGt83PO4rfN8I+rlYyOft6rkRaTlniv8/zLeXnFxaVzlUdsYYJSt3lMx4TU1C8zsWLxWaSR+81wf76RSa5t/L6wfPKir17bQN5jSdfi3YMzs/cBU4DozG5PyFQmvfbuiuJwVekOameW0XpFSej2PvgQ43cy+l1Janse9kRrsSJJSesTM7sNren8OfBWYVtjmOOBUvMZtCJ5HQ6usqhGb5mVnWqEnOl77MbL6Im8ZhgfPtax2HMzbzh6P16YOxGsf/liYZRrwTbzT1xeBZ1JKd+dpY4CPlcqO4bU3RfNK29wT+BHwfrymbAB+4wWvJQW/gVU8VVrfGOA8Mzu3MG4g8AyNe6u8ppRe9mKz2nlTfhw9r7BMMrP55GPSTRkYjtcYzu5B+sq2YPVjOwf4bGG4fJ6Xz8NqenKt2AJ4qnAuVtKwRf5/ZJXpxTSPyX//mvO7YlBhHfVU5imuc05h2nP03qKU0suF4Xl0f771RjmNL1On3DXhXKm2zUbKRU8067gPY1VFwBpRG06Rvukp/Jv8xqXPeimlBWb2Mfxx4ARg0xykzcRvqI26H38827CU0qN4kLMl/uisUXvgj56OqrTzwh/fDMVraHprGnBkbne0M3BZYdpVeC3qNimlYVTvpFS0Ag9AKt5T+P95/IbwydLx2CCldHo3690Wz+tauhwHM9sCf4R3Cl5DtBH+WLF4bK8CtjGzHfCajmmFaU/htSHFdG6UvCNW0ZuFbQ4GZuT1jsr59YPCNhfkv6MKyxf/r2z3qNJ2h6aUvlln35thdOWf/MVmFKuC3HplYDEeSGxdY71v1hhf9HRx+9nYPL5dnga2tK5RQzENC6pMH134vxIMbV06dkNSSlc2uP3yOseWpi2n9rkFtfN6hJkNKaW7cmwrX1J7s95ea9K50lPV9qOcp9B1/5t13LfFa4Bf62Xa36KAU6Rvug4YbN6hYUNzm5vZ5/L0Yfjj7cVAMrNP4+2KemIG/qinJjM7yswOsvxbkrmB/kTg4ZTSiz3Y1gS8/dz7gQ/nz7Z4oPSNHqa76Co8kD0Xb2O1oDBtGP54aLmZjcLbMtXzJ+ArZjY4N+4/vjIh1xL8AjjLzLYGMLOh5r9jWr7JvSUHwsPx9mC1zKBrp6Kh+LV5MfC6me0MHF5cIKW0FPgdHpTujLfhrLgMCPnYrWdm6+ROBvvUScNgvKZmSUrpFTP7IP6YsLK9Z/DHk6fn8jgcKP/czDnAFPNOPmZm65vZLrlWvJWOMrMdzDuTnIDXZF6fp9UsA/mYTgXONO9kZeadWLbPsyzEnzIMrrPtK4FxZnaEeaeynfCyfklT97C+6/FjNymX3ffhAVAlDdfhZeoE805SO+DtjwFIKS3Cn4xMtfzzN2a2sZl9zko/XVZNSulZ4Cbg7LzcJsDZwO9TSpVavD8Bh+RzZjje3rSoVl6vA5yRy9JYvLnIpXm7L5C/5Jj/0sJ2+FOU8nob7vzUoGacKz1VLX/+jAfk++Vz/HPAboXpzTrue+LXqDWmgFOkD8qPkfbAa74exW+at+KBGsAsPLC4F699OxAPQHpiFrDSzMbXmWcJ/uj2ETP7O952cCneFq4h5r2CDwDOSiktLH7wWtqPWDe9vWtJKS3D93tf/CeIir6Bt/lajrdB/U03qzsGvzm9iLeRm16a/mPgWuBaM3sJ79gxkfrX0aOA6TmdtVwOfCjfUEkpPVLY1lI8SKpW0zQN3+9ZhRs7OV8/juf5PPwY/o5SD9WilNIK/DifaWYr8BrVcvOMQ/Fg7hm8A1olP1/N67gI78g1LW9zPh5YDKqz781wIf6FYwlwMN4ms5Lf3ZWBE/FjPSPPczurApTf4DV0C817Eo8pLUtKaS7evu8YvIPG5cDklNLVzdq57uR93Qv/0vI3Vl0bfp6nL8U7Yh2M59G5wH+VVnM03kHvdjNbjrdNPgh/lNqIw/D8ewy/Xi0FjihMPwn/gvwcnsdXlZavlddP4eVtLn7tuREvYxVfwa9Fy/L+lgP9c/AvX0vN7KEG96WuZpwrvbBa/iT/GbXv4OX/RWAfvKNSJZ1rfNzNbGO8fJ/fy3R3YV0f74tIf5JrvSallHbLw+PxAGl0J9O1Nsq1onNTSpaHh+O9w0Op/V21ZSfinX4OrzdfX2Jme+NB8fqpQzcS83bCJ5XbD8vaz8yOxI9ts2so264vnCu9YWan4e2Hm/Lj+eo0JNKPpZRuxGsNpMlykLllg/OeT5NqEVrFzD6MtyV7AO9wcArw67XpBirSDm+XcyWl9G/NXJ8eqYtI0TzW7jf7dNJSvCPU29Um+GPpFcBdwF/xR3oi0pXOlSr0SF1EREREWko1nCIiIiLSUgo4RURERKSlFHCKiIiISEsp4BQRERGRllLAKSIiIiIt9f/cO4tTQwdy5QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 576x396 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "shap.summary_plot(shap_values_mxgb, X, plot_type=\"bar\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/navdeep/article-information-2019/models/MXGB_BEST_GRID_v2_HMDA_1575928031.3747108/mxgb_grid2_model_12\n"
     ]
    }
   ],
   "source": [
    "# Save the model\n",
    "model_path = h2o.save_model(model=best_mxgb, path=\"../models/MXGB_BEST_GRID_v2_HMDA_\" + str(time.time()), \n",
    "                            force=True)\n",
    "\n",
    "print(model_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "xgboost Grid Build progress: |████████████████████████████████████████████| 100%\n"
     ]
    }
   ],
   "source": [
    "# Train and validate a random grid of XGBoost models\n",
    "xgb_grid = H2OGridSearch(model=H2OXGBoostEstimator(distribution='bernoulli',\n",
    "                    # more trees is better if the learning rate is small enough \n",
    "                    # here, use \"more than enough\" trees - we have early stopping\n",
    "                    ntrees=10000,\n",
    "                    # learning rate annealing: learning_rate shrinks by 1% after every tree \n",
    "                    # (use 1.00 to disable, but then lower the learning_rate)\n",
    "                    # learn_rate_annealing = 0.99,\n",
    "                    # score every 10 trees to make early stopping reproducible \n",
    "                    # (it depends on the scoring interval)\n",
    "                    score_tree_interval = 10,\n",
    "                    # fix a random number generator seed for reproducibility\n",
    "                    seed = 1234,\n",
    "                    # early stopping once the validation AUC doesn't improve by at least 0.01% for \n",
    "                    # 5 consecutive scoring events\n",
    "                    stopping_rounds = 5,\n",
    "                    stopping_metric = \"AUC\",\n",
    "                    stopping_tolerance = 1e-4),\n",
    "                    grid_id='xgb_grid2',\n",
    "                    hyper_params=hyper_params,\n",
    "                    search_criteria=search_criteria)\n",
    "xgb_grid.train(x=X, y=Y,\n",
    "                training_frame=train,\n",
    "                fold_column=fold_column,\n",
    "                ntrees=100,\n",
    "                seed=SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get the grid results, sorted by validation AUC\n",
    "xgb_gridperf = xgb_grid.get_grid(sort_by='auc', decreasing=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "          col_sample_rate learn_rate max_depth         sample_rate  \\\n",
      "0                     0.9       0.08         6                 0.8   \n",
      "1     0.30000000000000004        0.1         6  0.7000000000000001   \n",
      "2                     0.9       0.06         6                 0.9   \n",
      "3                     1.0       0.09         5                 0.5   \n",
      "4      0.7000000000000001       0.04         8                 1.0   \n",
      "5                     0.5        0.1         8  0.7000000000000001   \n",
      "6     0.30000000000000004       0.08         6                 0.8   \n",
      "7                     0.4       0.04         9                 0.5   \n",
      "8      0.6000000000000001       0.04         7                 0.8   \n",
      "9                     0.4       0.08         5                 0.8   \n",
      "10    0.30000000000000004       0.04        10                 0.9   \n",
      "11     0.7000000000000001       0.08         9  0.6000000000000001   \n",
      "12                    0.9       0.04         7                 1.0   \n",
      "13                    0.8       0.05        10  0.6000000000000001   \n",
      "14                    0.4       0.07         5                 0.5   \n",
      "15                    0.2       0.09         6                 1.0   \n",
      "16                    0.2       0.06         7                 0.8   \n",
      "17                    0.8       0.09        10  0.7000000000000001   \n",
      "18                    1.0       0.09        10                 0.9   \n",
      "19     0.7000000000000001       0.09        10                 0.5   \n",
      "20     0.7000000000000001        0.1         3  0.7000000000000001   \n",
      "21     0.6000000000000001       0.05         4                 0.9   \n",
      "22                    0.9       0.04         4                 0.5   \n",
      "23                    0.1        0.1        10                 0.5   \n",
      "24                    0.2       0.03         7                 1.0   \n",
      "25                    0.9       0.06         3                 0.8   \n",
      "26                    0.2        0.1         3                 0.8   \n",
      "27                    0.4       0.01         6  0.6000000000000001   \n",
      "28                    0.2       0.06         4                 0.9   \n",
      "29     0.7000000000000001       0.08         2                 0.5   \n",
      "30                    1.0       0.07         2  0.6000000000000001   \n",
      "31                    0.1       0.09         3                 0.9   \n",
      "32    0.30000000000000004       0.06         2  0.7000000000000001   \n",
      "33                    0.1       0.02         6                 0.5   \n",
      "34                    0.1       0.01        10                 0.5   \n",
      "35                    0.1       0.01         6  0.6000000000000001   \n",
      "\n",
      "             model_ids                 auc  \n",
      "0   xgb_grid2_model_11  0.8249885080979954  \n",
      "1   xgb_grid2_model_12  0.8247223934698241  \n",
      "2    xgb_grid2_model_7  0.8243583326327336  \n",
      "3   xgb_grid2_model_24  0.8242440171166412  \n",
      "4   xgb_grid2_model_34  0.8240645819543483  \n",
      "5   xgb_grid2_model_20  0.8240278509292379  \n",
      "6   xgb_grid2_model_16   0.823937788011574  \n",
      "7   xgb_grid2_model_19  0.8237092291193193  \n",
      "8   xgb_grid2_model_33   0.823642685822769  \n",
      "9    xgb_grid2_model_3  0.8234500365846301  \n",
      "10   xgb_grid2_model_2  0.8234268562883731  \n",
      "11  xgb_grid2_model_27  0.8233271310731521  \n",
      "12  xgb_grid2_model_35  0.8232954634250521  \n",
      "13   xgb_grid2_model_4  0.8228114285404351  \n",
      "14   xgb_grid2_model_6  0.8226423383331863  \n",
      "15  xgb_grid2_model_22   0.822163829723465  \n",
      "16  xgb_grid2_model_31    0.82151035974039  \n",
      "17  xgb_grid2_model_26  0.8210874897197538  \n",
      "18  xgb_grid2_model_28  0.8209159260480484  \n",
      "19  xgb_grid2_model_36   0.820806903242695  \n",
      "20  xgb_grid2_model_18  0.8198402112570651  \n",
      "21  xgb_grid2_model_30  0.8180025899259131  \n",
      "22  xgb_grid2_model_13  0.8167518756877242  \n",
      "23   xgb_grid2_model_9  0.8158642321438193  \n",
      "24  xgb_grid2_model_29  0.8153264454410061  \n",
      "25  xgb_grid2_model_17  0.8153150223947837  \n",
      "26  xgb_grid2_model_25   0.815212050214744  \n",
      "27   xgb_grid2_model_1   0.815172571192665  \n",
      "28  xgb_grid2_model_14   0.813717237831632  \n",
      "29  xgb_grid2_model_15  0.8093904869915753  \n",
      "30   xgb_grid2_model_5  0.8079314070299111  \n",
      "31  xgb_grid2_model_21  0.8028519744319256  \n",
      "32   xgb_grid2_model_8  0.7983561088926685  \n",
      "33  xgb_grid2_model_32  0.7887743043571585  \n",
      "34  xgb_grid2_model_10  0.7856031955655638  \n",
      "35  xgb_grid2_model_23  0.7812513487133275  \n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(xgb_gridperf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8295507199389082"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Grab the top GBM model, chosen by validation AUC\n",
    "best_xgb = xgb_gridperf.models[0]\n",
    "\n",
    "# Now let's evaluate the model performance on a test set\n",
    "# so we get an honest estimate of top model performance\n",
    "best_xgb_perf = best_xgb.model_performance(test)\n",
    "\n",
    "best_xgb_perf.auc()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Shapley Importance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "First 5 rows of contributions:\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead>\n",
       "<tr><th style=\"text-align: right;\">  debt_to_income_ratio_missing.False</th><th style=\"text-align: right;\">  debt_to_income_ratio_missing.True</th><th style=\"text-align: right;\">  debt_to_income_ratio_missing.missing(NA)</th><th style=\"text-align: right;\">   term_360</th><th style=\"text-align: right;\">  conforming</th><th style=\"text-align: right;\">  loan_amount_std</th><th style=\"text-align: right;\">  loan_to_value_ratio_std</th><th style=\"text-align: right;\">  no_intro_rate_period_std</th><th style=\"text-align: right;\">  intro_rate_period_std</th><th style=\"text-align: right;\">  property_value_std</th><th style=\"text-align: right;\">  income_std</th><th style=\"text-align: right;\">  debt_to_income_ratio_std</th><th style=\"text-align: right;\">  BiasTerm</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td style=\"text-align: right;\">                          -0.0180966</td><td style=\"text-align: right;\">                       -0.000509719</td><td style=\"text-align: right;\">                                         0</td><td style=\"text-align: right;\">-0.00859936</td><td style=\"text-align: right;\"> -0.00181858</td><td style=\"text-align: right;\">        -0.379428</td><td style=\"text-align: right;\">                 0.112091</td><td style=\"text-align: right;\">                -0.0277947</td><td style=\"text-align: right;\">             -0.0613861</td><td style=\"text-align: right;\">          -0.191586 </td><td style=\"text-align: right;\">   -0.584845</td><td style=\"text-align: right;\">                 -0.232942</td><td style=\"text-align: right;\">         0</td></tr>\n",
       "<tr><td style=\"text-align: right;\">                          -0.0180955</td><td style=\"text-align: right;\">                       -0.00137637 </td><td style=\"text-align: right;\">                                         0</td><td style=\"text-align: right;\">-0.00324587</td><td style=\"text-align: right;\"> -0.00377074</td><td style=\"text-align: right;\">        -0.429348</td><td style=\"text-align: right;\">                -0.49926 </td><td style=\"text-align: right;\">                -0.0304844</td><td style=\"text-align: right;\">             -0.0943847</td><td style=\"text-align: right;\">          -0.646625 </td><td style=\"text-align: right;\">   -0.744191</td><td style=\"text-align: right;\">                 -0.932606</td><td style=\"text-align: right;\">         0</td></tr>\n",
       "<tr><td style=\"text-align: right;\">                          -0.0113302</td><td style=\"text-align: right;\">                       -0.000509159</td><td style=\"text-align: right;\">                                         0</td><td style=\"text-align: right;\">-0.00608778</td><td style=\"text-align: right;\"> -0.00139556</td><td style=\"text-align: right;\">        -0.389222</td><td style=\"text-align: right;\">                -0.217226</td><td style=\"text-align: right;\">                -0.361329 </td><td style=\"text-align: right;\">             -0.875725 </td><td style=\"text-align: right;\">          -0.0697521</td><td style=\"text-align: right;\">   -0.456006</td><td style=\"text-align: right;\">                 -0.553299</td><td style=\"text-align: right;\">         0</td></tr>\n",
       "<tr><td style=\"text-align: right;\">                          -0.0693226</td><td style=\"text-align: right;\">                       -0.000627809</td><td style=\"text-align: right;\">                                         0</td><td style=\"text-align: right;\">-0.0249719 </td><td style=\"text-align: right;\"> -0.00395645</td><td style=\"text-align: right;\">        -0.983011</td><td style=\"text-align: right;\">                -1.55428 </td><td style=\"text-align: right;\">                -0.218873 </td><td style=\"text-align: right;\">             -0.17173  </td><td style=\"text-align: right;\">          -1.15268  </td><td style=\"text-align: right;\">   -0.404977</td><td style=\"text-align: right;\">                 -0.781003</td><td style=\"text-align: right;\">         0</td></tr>\n",
       "<tr><td style=\"text-align: right;\">                          -0.0176021</td><td style=\"text-align: right;\">                       -0.000509159</td><td style=\"text-align: right;\">                                         0</td><td style=\"text-align: right;\">-0.00865142</td><td style=\"text-align: right;\"> -0.0013644 </td><td style=\"text-align: right;\">        -0.415674</td><td style=\"text-align: right;\">                 0.103324</td><td style=\"text-align: right;\">                -0.0284888</td><td style=\"text-align: right;\">             -0.0557718</td><td style=\"text-align: right;\">          -0.107839 </td><td style=\"text-align: right;\">   -0.370852</td><td style=\"text-align: right;\">                 -0.178833</td><td style=\"text-align: right;\">         0</td></tr>\n",
       "</tbody>\n",
       "</table>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "shap_values_xgb, expected_value_xgb = get_shapley(best_xgb, train, X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAApwAAAFeCAYAAAAog8+HAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xm8XdP9//HXjaFKEmNQRAaUnw6qfbdoayi++X6JVEt9Y4zQ+KIU9UXU1AiVRmjU0IqhQmIeWoKYWlG+pn5QWqWKDGhCkKSSaCPJ+f2x1mHnuMPJdXducvN+Ph73cc/Za++11t7n5pH3XWvtfRsqlQpmZmZmZmXp1N4dMDMzM7OOzYHTzMzMzErlwGlmZmZmpXLgNDMzM7NSOXCamZmZWakcOM3MzMysVA6cZmZmZlYqB04zMzMzK5UDp5mZmZmVasX27oBZRzJu3LhKv3792rsbZmZmS0pDPTt5hNPMzMzMSuXAaWZmZmalcuA0MzMzs1I5cJqZmZlZqRw4zczMzKxUDpxmZmZmVioHTjMzMzMrlQOnmZmZmZXKgdPMzMzMSuXAaWZmZmalcuA0MzMzs1I5cJqZmZlZqRw4zczMzKxUDpxmZmZmVioHTjMzMzMrlQOnmZmZmZWqoVKptHcfzDqMhvPm+x+UmZktlSonrFhGtQ317OQRTjMzMzMrlQOnmZmZmZXKgdPMzMzMSuXAaWZmZmalcuA0MzMzs1I5cBZIGi/ppPbux/Lgk15rSQdKmtSGXWqurSskjV4SbZmZmXVEpdwfv6yKiN3q3VfSBOCBiDi7vB4tPkkDgdMiYtP27ktzFudal0XSEOCbEbFre/fFzMysI/MIZ4kkrbQ019ceOsI5mJmZ2eLxCGdBddQSGAtMBAYAPwa6A48BB0fEVEkXA9sD20k6GXgjIjbP064rAR8A3wZuBI6UtCNwLrAFMBUYGRGj6uzPn4CewM7AOZLGAlcAXwFWBp4DjouIpyRtB1wKrCxpdq5mj4iYIOnzwPnAl4H3gWuBMyLigxb6UD2nhcCewHTgrIgYXdhne2AYsCUwA/gl8POIqEjaKV/TQ4AzgW5Al9oRYklfBC4Ats51/BoYFhELcvnXcr1b5GtyX0vXLx/XExgFbANUSJ/rfsCXgFOAToVr9cWIeFXSocCpua+3kx5qO7+e9szMzOzjPMLZvP7ADsCGwGrAUICIOBp4mBS8OkfE5oVj9gHGk8LK/0rqBdwD/ApYGxgIDJO0T519OBS4EFg9f+9ECl49gPWBp4HbJK0UEY8BRwCv5n51zmFzXeAh4LZ8LtsB/0EK0/X4b+BeYC3gcOBXkr4OIGlL4G5gRD7nvsDRwEGF41cAdieFyfVqK5e0OnA/8GA+p775vI8vlI8Hbsl9+BHwgzr7fg4wJbe7Dun6z4iIG3PZhMK1ejWH50tI13Gt3K/+dbZlZmZmjfAIZ/POjIi3ASRdBwyq45hHcpgBmCtpP+Dpwojg45JG5bpurqO+WyLi99X6SOFpSrVQ0mnAMcBmwF+bqGMA8GxhVPUNScOA4eQQ3YLHI2Jsfv2ApFtJwe1RUvC7OSJuz+Uv5hHgAcA1hToGR8SsJurvC8wDzo6ICvCCpOGkwDkC2AOYAwzP5X+UdCVwQB19n0cKsb0j4gXSiHBzBpCu+f35/TWSDq+jHTMzM2uCA2fzphZezwG61HHMpJr33UnTuEWvkKan67FIfZLWAX4O7ASsQZrqhjS62JRewDckzSxsayCNPC52H/L7Lxfq3lnSXoXyTsBrhfcLa97X6g5MzmGy6pW8HWCjRsprr2lTTgROB8ZJWo00SvrjiJjdxP4bAVGzrd62zMzMrBEOnK23sM7tr5Gmk4t603wAa66+YcBngG3yetIuwD9JAbKpfk0mrZfsW2ebtXo28v71Qt2/joijmjm+UhMWa70G9JDUUNiveI3eaKS8tk+NiojppBHgYyT1Jq3JPAk4g8av1RuN1N0TeLme9szMzOzjHDhbbxpQz6OHrgdOlzQAuI40Mng4cGQr2+1KmlqfIakzaVq8tl/rSuoaEf/M264hrSc9NPdhHilEfTYi7qmjzW3z0oCbgB2BvYHqo4R+CTwk6R7SWtUK8FmgW0Q8VOc53UW6YegUSSNIo6aDSTf7ANxJWr96oqSRwBeA7wP/bqliSf2BJ0mjsrNI574gF08DNpa0ckTMy9vGAPfkm6UeAvYl3XDkwGlmZtZKvmmo9UYCkjRT0vNN7RQRE0kjnEcD75ACzekRcVMr2z0DWDfX9RxpHeWCQvmDpBtdJua+7RgR04BvAd8hBa8ZwG9Io4j1uCmfwwzgSuCoiPi/fH5/Ia2xPI60BOEtYDTNT/EvIq/t7EMKsW+SblC6hrR0gIiYSVrn2T/34ULSTVj12JoUHGcDz5NushqRy24mjaJOy9eqVw7JPyQ9CeBd4L9ITxswMzOzVmqoVJqb6bTlXR7pmx8R9dwwtdxrOG++/0GZmdlSqXJCKRPbDS3v4hFOMzMzMyuZ13C2I0mnkB4+3pjdIuLhJdCHA/horWStpf5xQJI2punHQY2NiCOWZH/MzMzs4zylbtaGPKVuZmZLq/acUvcIp1kbumPz8fTr16+9u2FmZrZU8RpOMzMzMyuVA6eZmZmZlcqB08zMzMxK5cBpZmZmZqVy4DQzMzOzUjlwmpmZmVmpHDjNzMzMrFR+8LtZG/KD383MrDklPXy9PflvqZuZmZlZ+3PgNDMzM7NSOXCamZmZWakcOM3MzMysVA6cZmZmZlaqZSpwStpIUkVSzzr2HSLpgSXQrVaRdKmki9u7H0uzpekaSXpA0pD27oeZmdmyqMPdm1+vHFonAt0j4vU69h8InBYRm7ZF+xFxRFvU01FImkS6vmOr25bENZI0GpgfEYPKbsvMzGx5tUyNcNqyR9JK7d0HMzMza19L9QinpPWBy4AdgTeBc2vKDwOOBboDrwKDI+K+wi4NkkYCA4D3gYsj4me57Nn8/W+SKsDwiDiriX5sB1wKrCxpdt68R0RMkLRj7tcWwFRgZESMquPcRlMYWct9OAo4JNf1PDAwIl7M5SsBJwIHAxsAb+XzvUXSisApwEBgTeBp4NiI+EuhrRWAD4C9gDnACcALwOW5vQAOiIh/5GNWBYYCewOrA08CR0fEyy2c107AA/k8zgS6AV0kHQscCWwIzACuJY1oLpA0DtgYuELSpcCjEdGnkWvUA7gQ+Abp87wV+HFEvN9Cn7YGLgK+ACwAXgT6AocBB+R99s27rw4sBE4mfR6rAldT54NtzczM7OOW9hHOa0kBYWNgB1KgAj4Mm4NJgWFN4FTgNknFKe8dSEH1M8CewPGS9s9lW+Xvm0dE56bCJkBEPAYcAbya9+2cw2Yv4B7gV8DauX/DJO3TyvMdSAp46wCvkUJS1dnAgcA+QFdSCH8pl51ICtW7A+sDDwP3S+paOP57pIC2FnAWKWgOBb4LrAdUSAGxqhpEt811PgHcWeeI5Qq5L1vnugFeB3bLfd8TOBQYBBAR/YApwKB8bfvUVphD9V3ANKBH7tc3gPPq6M8lwH353NcDjgfmRcS5pJ+xqwuf6wLSdf5R7uf6wNuknyUzMzNrhaV2hFPShsDOwKYRMQuYJelMUnCANLI5NCKqI5V3S3oQ2JcUziCNOA6PiArwlKTLSKHuujbq5n7A0xExOr9/XNIoUpC6uRX1jYiIKfDhqOTY/LqBNNrWPyKey/u+nr8gjSYOL4yGDs196Atcn/f5fUTclcuvIYXkMdX1q5JuIY34IWkdYH+gR0S8mbedCRwHbAM8Use5DM6fGwARcWuh7BlJY4BdgBZHg7OvAZsB20TEHGCOpNOA30o6On/GTZlH+qWle0RMAh5voa0BwKiIeApA0jDSLxxmZmbWCktt4AQ2yt8nF7ZNLLzuBVwi6cLCthX5KIQBTK4JIpNIU8ptpXtNnwBeIY2MtcbUwus5QJf8uhuwGh+NaDbbj4hYmG/C6d5Y3RExV1Jte3ML7fXK35/L+1WtVFNnUxaSRmg/JGk/0shib9LntDItB7+i7sD0HDarXgFWIV2ft5o59hDgdOARSR+QgvyZETG/if03Iv2sAB9ez8lN7GtmZmYtWJoD5xv5ew9SsADoWSifDPwkIpobSewhqaEQOnvyUSBduJj9aWz/10hTx0W9qQlbbWA6KRBuBvy9iX70rL6R1Cm/b20/quFqs4iY3orjK8WgL6k7KeTtBYyPiHmSzgOKabalz+M1oJukVSNibt7WG/gX6fo0KSImkqbwkfQF0ij5RODXTbT7BotezwbSz6GZmZm1wlIbOCPidUkTgHMlHQJ8GjijsMtIYIikv5NuAFoF+ArwdnVqmbR288R849DnSVPGx+ey6aSwsRmLjoo2ZRqwrqSuEfHPvO164HRJA0jT9F8GDifdHNNmIqIi6ZekazGFdEPRhsBaeYp9NHCSpD+QRuYGkz7bu1rZ3luSrgN+Kem4iHhD0hrAt4D7I2J2C1XU6kxaLzwd+EDStsBBpJuWqqaRPoumPAm8DJwv6X+BNUhrUa9qYTodSQfnfv8DmAnMJ60Nrra7raROEVENn2NI1/o3wJ9JN1itX+/JmpmZ2aKW9puG9gc+RRrdehi4ploQEZeT7g6/inTX8xTStGnxppaHSaFzGnAn8Avy+s18Z/PpwPWSZko6tYW+PAjcD0zM+++YR852B44G3iEFldMj4qZPctJNOBW4Cfgt8B4wAajeIDWCFH7vI90ktTPQpxCMW+Mw4G/ABEnvkYLXPqSbixZLRLwA/AS4nRT4TuajtaVVZwMHSpohaXwjdcwH9iBNd08hBdAnSGGwJTuT1vDOAR4j/QyMyWVXkJYrvJM/1xVIP2cXAeNI13Nd4A91n7CZmZktoqFSWez8YGZNaDhvvv9BmZlZkyonLLWTy61V12MDl/YRTjMzMzNbxnW4mN1akrYHPjaVm50TEecsZn2nkB7G3pjdIuLhxalvaVJ4+H2thyNityXaGdr+szMzM7O25Sl1szY0bty4Sr9+/dq7G2ZmZkuKp9TNzMzMrP05cJqZmZlZqRw4zczMzKxUDpxmZmZmVioHTjMzMzMrlQOnmZmZmZXKgdPMzMzMSuXncJq1If9pSzOzjq8D/nnKT8LP4TQzMzOz9ufAaWZmZmalcuA0MzMzs1I5cJqZmZlZqRw4M0mzJW3X3v1YXnzS6y3pCkmj27BLzbX1sqSBS6ItMzOzjsi3WWUR0Xlx9pdUAbaPiEdK6lKr5BA2PyIGtXdfmrO417sMkiYAD0TE2e3dFzMzs47MI5wlkrRSG9bVIGmZ/wWhLa+JmZmZLRuWuQAjaRJwGbALsA0wCfifiHg0B7JTgIHAmsDTwLER8Zc66v1wxDJPn54GXAicBKwG3AT8ICIWSHo2H3afpIXADRExKPft18C3gK8Cg4AbJB0JHAesD7wAnBgRD9fZp+OAg4DPAd+StBpwDvBZYD7wO+CYiHhL0knAAfnYfXM1q+c+fwc4HdgEmAqcHRHX1tGH6jn1Ab4EvAgcGRF/LOxzGHAs0B14FRgcEfflsiHADqTP4qD8fbfaEWJJewNnAD1Jn+mQiPhNoY1DgVOBbsDtpOd+za+j/7sCI/J5zwP+FBG7SroY2B7YTtLJwBsRsXkOxMOBA4GFwMiW2jAzM7PmLasjnIcCxwCrA/cDV+ftJwIDgN1J4e5h4H5JXVvRRg9gPVJQ+SqwD7AvQERslffpExGda6avDwOOB7oAt0vaDzgr92tt4HLgHkk96uzH94H+QGfgGeDfwNGk4PUFYAPgF7lf5wLXAlfnfnXOYfM/gCtJ4XUt4GDgYkk71NmHI0iBci3gFuDu6jXNYXMwKeiuSQqFt0natHD8DqSQ2x3Yu7ZySV/P/T6ZdI1OAa6XtE0u3x64JPdjLdJn3r/Ovl9D+sVhdWBD4GyAiDia9PNxVr5Om+f9Twb2AL4O9CIF4Ho/KzMzM2vEMjfCmY2KiOch3TwCHCdpdeAQYHhEvJjLhpJGGfsC1y9mG+8DZ0TEAuBlSb8DRApGzbk8Ip6p1iHpkNzfJ/K2KyUNAvYHhtXRj/Mi4pX8egFQXDM6TdK5pBHI5hwL/KIwqvqkpLGkEPyHOvpwZUQ8BSBpOPADUii7Ltc9NCKqo753S3qQFM6rayMnR8T5+fW8RuofCNwaEePz+7sk/Yb0i8UTuZ+3RMT9ufwaSYfX0e9qe5sA60XENGBCC/sPAH4WES/n8z2BFPrNzMyslZbVwDm18HpO/t6FNII2sVoQEQvzlHD3VrTxVg6bxXa61HHcpJr33UnT8UWvLEafFqlP0ldIU+pbAauSppZbugGnF2k6/vjCthVII3yL1YeIqEiaAmxUqPsSSRcW9l8ReL3wfnIL9XcHnqrZ9grw5fx6IyBqyidSnz1JI6Z/ljQduCwiLmhm/41Y9HznSHqrzrbMzMysEctq4GzKa6QpUAAkdcrvXyuhrab+ZvbC5vqU9QbG1dlObX03kKa194mIf0rao6au2v0hBb7RETGizjZr9ay+kNQAbMxHgXIy8JOIuLmZ4xvrU1FT16j6ub3RSHlP4OUW6iWPvPbP/f4mad3tcxHx+yb6tUhbec1st5baMTMzs6Z1tMA5GjhJ0h9Io1SDSed4VwltTQM2Y9Ep7qb69AtJd/DRjTNfAvZrZbtdgVnAe5I2Jq05rO3XtpI6RUQ1UF0AjJb0OPAoaXTzC0BDRNSOHDbm0DzF/WfgR6SR1eo1HQkMkfR34FlgFeArwNvVpQ11uBp4QNIY4AHSDUp7ATvl8jGkda+jgYdI0/Xb0ELglLQy6TrfFRFvS5pBCpnVketpwKY1h40BTsyPTPoHcC7L7lpnMzOzpUJH+490BGmt5n3Am8DOpBt7/llCW6cCQyXNkDSqqZ0i4jrgTGAs8A5wJLB7RLQ0zdyU/yGtS30PuA2oHVm8gnRX/TuSZkpaId8xfhjp+rxNWpIwkpan4qsuI914M4N0s07fiJiVz+9yUii7KpdPId0NX/fjjyLi/0g3Mp2X6zgXODAiHs/lDwE/zOf2LvBfwI11Vt8feFHSbOAO0mjsQ7lsJKB8nZ7P24YB9wKPk6btp9DykgAzMzNrRkOl0tTMsNmHj0U6LSLGtndflgUN5833Pygzsw6uckJHmyD+RBrq2amjjXCamZmZ2VJmuYnoecq0secpTo6Izy3p/gBIGk96+PjHLKk//SjpUtJDzhuz5ZLowych6QCgqSUNh9fzcHszMzMrl6fUzdqQp9TNzDo+T6kvoq4pdV8xszZ0x+bj6devX3t3w8zMbKniNZxmZmZmVioHTjMzMzMrlQOnmZmZmZXKgdPMzMzMSuXAaWZmZmalcuA0MzMzs1I5cJqZmZlZqfzgd7M25Ae/t54fpGxmtkzy31I3MzMzs/bnwGlmZmZmpXLgNDMzM7NSOXCamZmZWakcOM3MzMysVA6c9iFJ4yWd1N79WNpIukLS6Pbuh5mZ2bLKzyGxD0XEbu3dhyVJ0hDgmxGxa3v3xczMrCPzCKeZmZmZlcojnPYhSROAB4CxwERgAPBjoDvwGHBwREzN+3YGhgB7Ad2A14DDI+JhSasCw3LZp4FHgGMiYkqhnaeBXsCuwFvA/5AeHnsBsDHwO2BARLyXj1kbOBfoA6wCPAj8MCLebOGcegKjgG2ASj6v/YAvAacAnSTNzrt/MSJelXQocGo+r9tzv+YvxqU0MzOzAo9wWnP6AzsAGwKrAUMLZVeSQtwuQFfg28DUXDYS2DZ/9QDeBsZJWqFw/EHAz4A1gBuBMaTQuQPQE9gcOAZAUgPwW1Jg/Hyu8z3gujrO4RxgCrAesA4wEJgRETfmsgkR0Tl/vSppe+AS4AhgLeD+fB3MzMyslTzCac05MyLeBpB0HTAov14X+G/g8xExMe/7ci7rBBwM9IuIN/K244B3ga+RRkoBboqIJ3L5WNJI6oiIeDdvuxNQ3vcr+WvXiPh3Lj8JeFvSRhHxejPnMA9YH+gdES8Az7VwzgOAWyLi/vz+GkmHt3CMmZmZNcMjnNacqYXXc4Au+XXP/P2lRo7pBnyKNHUNQETMJk2bd2+i7rlNbKu21yvX+aakmZJmAq8A/yJNvzfnxNyXcZKmSrooLwdoykbApJptExvZz8zMzOrkEU5rjUn5+2bAX2vKpgP/JoXS6qhnZ2Bd0jrP1phMCrxrRcTCxTkwIqaTpuaPkdSbtCbzJOAMoLG63uCjQF3Vk3wuZmZmtvgcOG2xRcRbkm4BfilpICkQbpLLXpZ0DXCWpL8CM4HzgReBJ1vbJPAscKGkn0TEO5K6AbtExA3NHSipf253EjCLNMW+IBdPAzaWtHJEzMvbxgD35OduPgTsS1qr6sBpZmbWSp5St9Y6FPgTKZS9Rxo5XD+X/YgUEv9IumHnM8C3I2JBI/W0KI9q7km6W/wpSe8BjwM71XH41rmPs4HnSXfHj8hlN5NGXaflqfpeEfEQ8EPgCtK60/8i3dRkZmZmrdRQqVTauw9mHUbDefP9D6qVKid4wsXMbBnUUM9OHuE0MzMzs1J5SMGWaZI25uM3LlWNjYgjlmR/zMzM7OM8pW7WhsaNG1fp169fe3fDzMxsSfGUupmZmZm1PwdOMzMzMyuVA6eZmZmZlcqB08zMzMxK5cBpZmZmZqVy4DQzMzOzUjlwmpmZmVmp/BxOszbUEf60pf/EpJmZLQY/h9PMzMzM2p8Dp5mZmZmVyoHTzMzMzErlwGlmZmZmpXLgNDMzM7NSOXBau5C0k6T57d2PekmaL2mn9u6HmZnZssiBczkiaaCkl9u7H0sTSZMkHdje/TAzM+vIHDiXMEkrLU/tmpmZmfkJz21A0iTg10Af4EvAi8CREfFHSaOBlYAPgG8DNwJHStoROBfYApgKjIyIUbm+nYAHgO8DQ4EuwB3A0RExO++zdj6+D7AK8CDww4h4s6ZP3wK+CvwU+AmwsqTZuet7AMOBGyJiZOF8hgLfiIhdmjnnEcBmEfGdwradgHHA+kAFGAt8HVgVeBkYHBH3N1HfaGB+RAyqua6nRcTY/H57YBiwJTAD+CXw84ho9mHrkvbN574RMBe4JyIOljQO2Bi4QtKlwKMR0UdSF+BioB/wHnBGc/WbmZlZ8zzC2XaOAI4F1gJuAe6W1DWX7QOMB7oB/yupF3AP8CtgbWAgMEzSPoX6ViAFni8C/w/4LPBzAEkNwG9Joe7zQA9SMLqupk+HAceTAusFuY+vRkTn/DUBGEUKtuS6OwGHAJe3cL5XAbtL6lbYdghwU0TMIf1s3QZsls/xeuDWmv3rJmlL4G5gBOk69gWOBg5q4bhVgTHAURHRBegNXAEQEf2AKcCgfD365MMuyP3eknT99yR9HmZmZtYKHuFsO1dGxFMAkoYDPyCNIAI8EhE35tdzJe0HPB0Ro/O2xyWNAgYBNxfqHBwRs4BZks4A7pR0BPBl4CvArhHx79zmScDbkjaKiNfz8ZdHxDP59fuSGuv3DcBISdtGxOPAfwKfBn7T3MlGxF8lPQMcmI/vAnyPNOJKHokdWzhkhKTBpNHWu5uruwk/AG6OiNvz+xclXQwMAK5p4dgPgC0k/Ski3gUebmrHHLgPAPpGxLS8bTDw3Vb02czMzHDgbEuTqi8ioiJpCmkKd5GyrDswsWbbK6SRtKLJNfV/ClgH6JVfv1kTIv9FmiJ+vXBMsyJirqSxpLD7eP5+TTXItuAq4EhgJPDfwOsR8X8Akj5NGo3cPfd5IWmktVUjnKRz3lnSXoVtnYDXmjson9/upJHen0p6FTg/ImpHg6u6ka7tpMK22s/KzMzMFoMDZ9vpWX2Rp7yrwW9LUtgqeo0UxIp68/Hw1IMURKv1/xt4mxRE5wBrRURt3UW1ZU3tOwr4P0k/JU3jf6mZOouqo6NfJi0LuKpQdjywA7ALMCmH8LeBhibqeo8UTAGQtCKwbqF8MvDriDiqzr59KC8dmCBpBdI62lslPRERr/Dxa/I2MI90vYvX3szMzFrJazjbzqGSvpzvBj+RdKPMXU3sez3wFUkDJK0o6WvA4cCVNfsNk9RV0rrAEGBMDpgBPAtcmG8eQlK3fHNMc6YB6xbWlgIQEc8BzwO3Ak9GxF/rOeGImEmaej8b2Ba4ulDclRSQ3yHdqHQGsEYz1T0F7CKpl6RPkW5yKt5Z/0tgX0n9JK2Ur9uW+earJklaT9LeklaPiAXAzFy0IH+fRlqvWT2nBaS1sGfmY7sCP2uuDTMzM2ueA2fbuQy4kHT3dH/SGsBZje0YERNJI5xHkwLZGOD0iLipsNsCUmD9M/A34FXSqCE5dO5JGi18StJ7pOnwnVro44PA/cBESTNrwtooYGtavlmo1lXAbsC9ETG1sP3npHD3D9JI4Vyan+K/lnQn/tN5/ynAG9XCiPgLaU3scaS7+t8CRtPyFH0n4ChgUr5OlwAHR0S1L2cDB0qaIWl83nYsaRr9RdL1H8dHAdXMzMwWU0Ol0uwTZawOtY/vaYP6dgIeiIgltuQht/lbYIOImLuk2u1oGs6bv8z/g6qc4JU2ZmZWt6aWyi3CI5yGpFWAE0h3tTtsmpmZWZvyUMZyLt/1PZY0lf3TmrLxwPaNHRcRncvvXX0knQKc0kTxbhHR5GOQzMzMrHyeUjdrQ+PGjav069evvbthZma2pHhK3czMzMzanwOnmZmZmZXKgdPMzMzMSuXAaWZmZmalcuA0MzMzs1I5cJqZmZlZqRw4zczMzKxUfg6nWRsq409b+k9NmpnZUszP4TQzMzOz9ufAaWZmZmalcuA0MzMzs1I5cJqZmZlZqRw4zczMzKxUDpxmZmZmVioHTjMzMzMrlR/wZ60maQLwQESc3d59qZL0U2B/YG3gX8AfgOMjYkphn02A84Cd86YXgO0j4oNcLuCXwOeBqcBPImLsEjsJMzOzDsYjnNZuJK1UQrVjgC9FRFegJzAFuKHQZjfgYeBZYGNgLeBoYEEuXx0YD9wKrAkcAVwqabsS+mpmZrZc8AintYqki4Htge0knQy8ERGbSzoMOBboDrwKDI6I+/IxQ4AdgKeBg4CnJR0JTAQGAoOBHsBDwAH5/aHAQuCsiLikpX5FxIuFtw0GMJ7lAAAgAElEQVT52M0L244HpkTEkOJhhdd7AXOBcyOiAtwv6TfA/wCPtdS+mZmZfZxHOK1VIuJo0kjhWRHRuRA2B5PC4prAqcBtkjYtHLoDaZq6O7B3YfvewDdJo449gSeAV4ANgEOACyRtXE/fJO0vaRYwmxR+hxSKvwW8JukuSe9Kek7SAYXyrYBnctisejpvNzMzs1bwCKe1pWOBoRHxbH5/t6QHgX2B6jrPyRFxfn49Ly2XBFJwfRdA0p1A34i4PJeNlzQD2Jo0Rd6siLgOuE7S+sD3gT8XitcBvgr0B/YkBdBxkiZHxCNAF2BWTZUzga4tnr2ZmZk1yoHT2lIv4BJJFxa2rQi8Xng/uYljpxZez615X93WZXE6ExHTJF0OvCpp4xxo3wMei4hb8m73S7oH+DbwSC7vWVPVGsA/F6dtMzMz+4gDp30SC2veTybd0X3zYhxTthWB1UhT8+8CfwI2bWS/6hT6s8B3asq2ztvNzMysFRw47ZOYxqLhbSQwRNLfSQFtFeArwNs1N/OUQlIn4AfATRHxlqSNgIuASUC1/VHAw5K+A9wB7Aj0AYbn8t8A50o6EfgF6caovYD/KLv/ZmZmHZVvGrJPYiTpsZUzJT2f11yeC1wFzCCttzwdKOPxR03ZHfiLpDmkG4/mArtGxHyAiHic9JzO4aTp84uAgyPisVw+M9exD2kt5+XAEdVyMzMzW3wNlUql5b3MrC4N581v839QlRM8EWFmZkuthnp28ginmZmZmZXKQye2TJF0KXBgE8VbFv+EZXu4Y/Px9OvXrz27YGZmttRx4LRlSkQcQfpzk2ZmZraM8JS6mZmZmZXKgdPMzMzMSuXAaWZmZmalcuA0MzMzs1I5cJqZmZlZqRw4zczMzKxUDpxmZmZmVir/aUuzNtSWf9rSf9LSzMyWAf7TlmZmZmbW/hw4zczMzKxUDpxmZmZmVioHTjMzMzMrlQOnmZmZmZXKgbMdSZog6bT27oe1TNLLkga2dz/MzMyWRQ6ctsyRtJOk+W1Ul0O/mZlZyRw4zczMzKxUfrL0UkLSF4ELgK2BGcCvgWERsSCXXwXsCqwBvAacHRHX5bKdgAeAA4BzgHWAe4HvR8R7LbS7FXAh8DlgBeBx4OiIeCWXj87bPwD2AuYAJwAvAJcDWwABHBAR/8jHrA2MBPrkZu4FfhQR7+byScBpETE2v+8JTAS6R8TrhTb/BeyT2xwaEaMkbQCMB1aQNDvXf1REXN3MOe4KjAA2AeYBf4qIXSVdDGwPbCfpZOCNiNhc0krAcOBAYGE+FzMzM2slj3AuBSStDtwPPAisD/QFDgWOL+z2CPAlUuAcCoyWtGWhfAVSwNsK+CwpuB5TR/MVYAiwIdATmA2Mrdnne8CtwFrAWaSgORT4LrBeruPMwv7XAmsC/y9/rQOMqaMvtW2Oy23+ELhYUo8cancDFkRE5/zVZNjMriGF6tXzeZ4NEBFHAw8DZ+V6Ns/7nwzsAXwd6EW6Lj0Ws/9mZmaWeYRz6dCXNPJ2dkRUgBckDScFzhEAEXFlYf8bJJ0A7AT8tbD95IiYDcyW9FtALTUcEc8V3v5b0pnAnyWtGhFz8/bfR8RdAJKuAX4FjImI1/O2W4DD8usNgP8EPhsRM/K244EXJX0mIqbWeU1+HxF35Ne3SZpJCtyT6zy+aB5pdHO9iJgGTGhh/wHAzyLiZYB8rb/finbNzMwMB86lRXdgcg6bVa/k7UjqRBqF7E8aAa0AqwHdCvsviIjphfdzgC4tNSxpE1Ko3SbvX+1DNz4Kdx+GxIiYK2mRbcDcQlvd8/eJNedSLas3cNbuV9f5NGFP4BRSkJ4OXBYRFzSz/0bApOqbiJgj6a1Wtm1mZrbc85T60uE1oIekhsK23nk7wH7AIGBvYM2IWAN4Fmjgk7sUeA/4YkR0Bb6Rt7e27mqfexa29a4pe48UmKs2WMw2Fi7OzhHxbET0B9YFDgeGSdq5mbreoNB/SbXh3szMzBaDA+fS4S7gU8ApklaWtDkwGKhOo3cF5gPTgU6SDiWt1WwLXUmjhzMlrUNam9lqeY3lfcD5ktaQtCZwPjC+MJ3+FLCfpM6SugGnL2Yz00g3DfVqacd8PQ+WtE4eQZ5BCpkLCnVtWnPYGOBESZtI+jRwLv63YmZm1mr+T3QpEBGzSDf87Aq8Sbqr+xrg53mXq4EngJdJo29bkm52aQs/It2p/c9c551tUOeBpFHMvwEvAjNJ6yKrTiMFvqmk9ZQ3LE7lEfESaR3pk5JmSjqohUP6k9aQzgbuAH4SEQ/lspGAcj3P523DSJ/B46SlAVNo3dpRMzMzAxoqlUrLe5lZXRrOm99m/6AqJ3iJtZmZLfXqWoLnEU4zMzMzK5WHUDo4SduTHpTemHMi4pwl2Z8ySDoAGNVE8eERce2S7I+ZmZktylPqZm1o3LhxlX79+rV3N8zMzJYUT6mbmZmZWftz4DQzMzOzUjlwmpmZmVmpHDjNzMzMrFQOnGZmZmZWKgdOMzMzMyuVA6eZmZmZlcrP4TRrQy39aUv/uUozM+tg/BxOMzMzM2t/DpxmZmZmVioHTjMzMzMrlQOnmZmZmZXKgdPMzMzMStUhA6ekCZJOa+9+LE0kVSR9s737sTgkPS+pf3v3A0DSfEk7tXc/zMzMlkV+RktJJA0ETouITdu7L0s7ST2BiUD3iHi9uj0iPrcE2p5E+pzGlt2WmZnZ8qpDjnDa0kPSSu3dBzMzM2tfHX6EU9IXgQuArYEZwK+BYRGxIJdfBewKrAG8BpwdEdflsp2AB4ADgHOAdYB7ge9HxHvNtLkdcCmwsqTZefMeETFB0o7AucAWwFRgZESMauEc+gJXARtGxAd5W2dgGtA3Ih6SdA6wL7Au8CZwUURc0ER9A6kZfZU0GpgfEYPy+42BnwPfBCrAOOB/mzvvfNwQYAfgaeCg/H235q4z8Gz+/jdJFWB4RJxVO/rYmmuXj9sX+AmwETAXuCciDpY0DtgYuELSpcCjEdFHUhfgYqAf8B5wRkttmJmZWdM69AinpNWB+4EHgfWBvsChwPGF3R4BvkQKQkOB0ZK2LJSvAPQBtgI+SwquxzTXbkQ8BhwBvBoRnfPXBEm9gHuAXwFrAwOBYZL2aeFU7gHm5/5X7UMKnH/I7/9KCoddgMNyvf/ZQr2NkrQK8PtcZy9gS1JY+0WdVexACoTdgb3ztuau81b5++b5Wp3VSJ9ade0krQqMAY6KiC5Ab+AKgIjoB0wBBuV2++TDLgA2y+f9RWBP0s+BmZmZtUJHH+HsC8wjjaZVgBckDScFzhEAEXFlYf8bJJ0A7EQKW1UnR8RsYLak3wJqZX/2A56OiNH5/eOSRgGDgJubOigiFkgaAxwC/DZvPgS4Kp8XNWsQfy/pLmAX0ojs4toDaIiI6sje+5JOBx6VdFh1dLgZkyPi/Px6Xu5fPde5Oa26dtkHwBaS/hQR7wIPN7WjpE6kEe2+ETEtbxsMfLfOfpqZmVmNjh44u5PCT/HvW7+St1fDxRCgP2kEtAKsBnQr7L8gIqYX3s8hjSK2tj8Ta7a9QhpBa8lVwHOS1s3tfx3Yv1oo6RjSyOZGpL9r+mngukbqqUcvYGNJM2u2V0jX6Y0Wjp9cfFPndW5Jq65dRMyVtDvpl4yfSnoVOL8wnV+rG/ApYFJhW227ZmZmthg69JQ6aa1gD0nFPyzfO2+HNGo2iDTtu2ZErEFaT1jXH6JvwcIm+tOzZluxP02KiBeBp4ADSdPJD1Tv6Jb0DWA4cDiwTj6PcTR9Hu+RAl/RBoXXk4GXImKNmq9VIqKlsAkfP/eWrnNj16rWJ7l2EyLi26Q1uGcDYyVt0kTbb5NGZYtt1bZrZmZmi6Gjj3DeRVqPd4qkEaSRu8FA9UaTrqS1kdOBTvlmmq2AO9ug7WnAupK6RsQ/87brgdMlDSCNPn6ZFBKPrLPOq0jrR7sAJxa2dwUWkM6jkm8y2o2mp5r/lPu2B3A3aZRwB6A6LX8naTTwFOAiYDYpkH4tIn5TZ1+LWrrO00nBbzPg9cYqoJXXTtJ6pLWtD0TErMKobXVZwLTcLvDh8oXrgDMl/QV4H/hZ/adqZmZmtTr0CGdEzCLd8LMr6c7te4FrSHdfA1wNPAG8TJom3pJm1vctpgdJNyxNlDRT0o4RMRHYHTgaeId0M8vpEXFTnXXeQBrV6wzcXthePa8nSSN03wOaDIYR8QpwLHAZ8C7wX8CthfK5wM6k6/EiMAv4Hemmn9Zo9jpHxPvA6cD1+Vqd2kifW3vtOgFHAZMkvQdcAhwcEZNy+dnAgZJmSBqftx1LmkZ/EfgzabS4pXWrZmZm1oSGSqXS8l5mVpeG8+Y3+w+qckJHn1QwM7PlTF3LEDv0CKeZmZmZtT8Pt7SSpO2B8U0UnxMR5yxmfacApzRRvFtEtNVU/yfS1ufdFpaVa2dmZra88pS6WRsaN25cpV+/fu3dDTMzsyXFU+pmZmZm1v4cOM3MzMysVA6cZmZmZlYqB04zMzMzK5UDp5mZmZmVyoHTzMzMzErlwGlmZmZmpXLgNGtD3/7bbu3dBTMzs6WOA6eZmZmZlcqB08zMzMxK5cBpZmZmZqVy4DQzMzOzUq3Y3h2w5Yekk4EfAasB34qIPy6hdi8F5kfE0UuiPTMzM1tUQ6VSae8+2HJA0kbAFODzEfHX9u5PWRrOm1+pnODf48zMbLnRUM9OnlK3JaUnsPCThE1JK7Vdd8zMzGxJ8QinLUJSZ2AIsBfQDXgNOBx4ChiWt38aeAQ4JiKm5OMm5H16An2At4DjI+J2Sf2B0cAqwBzgzYjYRNLawMi8P8C9wI8i4t1c5yTg18C3gK8Cg4AtgO2BAA4l/dL0U+BW4Kq830vAgRHxQq5nNGlKfVB+XwGOAg7J9T0PDIyIF3N5F+ASYA/gPeD03I9dI2JCc9fPI5xmZrac8QintcqVwDbALkBX4NvAVFIw3DZ/9QDeBsZJWqFw7MHA+cDqwMXA1ZJWjYgbgd2ABRHROSI2yftfC6wJ/L/8tQ4wpqY/hwHHA12A2/O2HYC/A+sDBwIjcr+PAtYCXgAubOE8BwJ75zZfAy4qlP0C6E0Ko18A+gIrYGZmZq3ioRj7kKR1gf8mrbOcmDe/LKkTKUz2i4g38r7HAe8CXwMey/veGBGP5vLLgJ8DmwHPNtLWBsB/Ap+NiBl52/HAi5I+ExFT866XR8Qz+fX7kgBeiogr8rbxkt4B7i2MaF5HCrPNGVEYnR0NjM2vVwAOAHaLiLfytlPydTEzM7NW8AinFfXM31+q2d4N+BRQDaFExGzStHn3wn5TC+Vz8ssuTbRVPW5iYdsrNWUAkxo5dmrN+7k12+Y2025jdcwp7L8OsDIwuVBefG1mZmaLyYHTiibl75vVbJ8O/JuPAml1ree6pOno1qge17OwrXdNGcDCVtbfWm8D80jLBqo2XsJ9MDMz61A8pW4fioi3JN0C/FLSQNLIXnW95TXAWZL+CswkrdV8EXiylW39Q9J9wPmSDiYtOj4fGF+YTl/iImJBnpIfIunPwL9INyWZmZlZK3mE02odCvwJeIh0h/btpJtzfkS6M/yPpOdpfgb4dkQs+ARtHZjb+BspvM4EBnyC+trKsaRzfAn4C3A/UCGN8pqZmdli8mORzFogaXNSIN4wIv7R3L5+LJKZmS1n6noskv9nNKshqTdpVPcJ0k1EI4E/tBQ2zczMrHGeUjf7uFWAy4BZwJ9Jd73v3649MjMzW4Z5hNOsRv7zm59v736YmZl1FB7hNGtDd2w+vr27YGZmttRx4DQzMzOzUjlwmpmZmVmpHDjNzMzMrFQOnGZmZmZWKgdOMzMzMyuVA6eZmZmZlcqB08zMzMxK5cBpZmZmZqVy4DQzMzOzUjlwmpmZmVmpHDjNzMzMrFQOnGZmZmZWqlYFTkkbSapI6lnHvkMkPdCadpYESZdKuri9+7E0a89r1BZtSzpA0rNt1SczMzNbPCu2dwdyaJ0IdI+I1+vYfyBwWkRs2hbtR8QRbVFPRyFpEun6jq1ua89r1BZtR8S1wLVt0B0zMzNrBU+pLyckrdTefTAzM7PlU10jnJLWBy4DdgTeBM6tKT8MOBboDrwKDI6I+wq7NEgaCQwA3gcujoif5bLqVOffJFWA4RFxVhP92A64FFhZ0uy8eY+ImCBpx9yvLYCpwMiIGFXHuY0G5kfEoPy+AhwFHJLreh4YGBEv5vKVgBOBg4ENgLfy+d4iaUXgFGAgsCbwNHBsRPyl0NYKwAfAXsAc4ATgBeDy3F4AB0TEP/IxqwJDgb2B1YEngaMj4uUWzmsn4IF8HmcC3YAuko4FjgQ2BGaQRv5Oi4gFksYBGwNXSLoUeDQi+jRyjXoAFwLfIH2etwI/joj3W+jTEGD7fI6Hkn7h+Wk+/irgq8BLwIER8ULhms2PiEGSGoCz8zl1Ad4Bzo+IiyStSfoZ3Zn0c/06cEREPFw7Ki5pAvAU0BPoQ/oMj4+I23N5A/Bj4AfAqsDVwBeBhyNiSHPnaGZmZh9X7wjntcACUhjZgRSogA/D5mDgAFLIOhW4TVJxynsHUlD9DLAncLyk/XPZVvn75hHRuamwCRARjwFHAK/mfTvnsNkLuAf4FbB27t8wSfvUeX61BpIC3jrAa8BFhbKzgQOBfYCupBD+Ui47kRSqdwfWBx4G7pfUtXD890gBay3gLFLQHAp8F1gPqJACYlU1iG6b63wCuLPOEcsVcl+2znVDCmK75b7vSQp+gwAioh8wBRiUr22f2gpzqL4LmAb0yP36BnBeHf2B9LPw93wuBwIjgCtJIX8tUvi+sIlj/4MU9LeJiC7A14BHctmJpHDYA1iDdD2bW6JxMHA+KcRfDFydwz3AQaRfoPqRrtvU3G8zMzNrhRZHOCVtSBo12jQiZgGzJJ0JVEcwjwWGRkR1pPJuSQ8C+5LCGaT/sIdHRAV4StJlpFB3XRudx37A0xExOr9/XNIoUpC6uRX1jYiIKfDhCNvY/LqBFIz6R8Rzed/X+SjYHEI6z+po6NDch77A9Xmf30fEXbn8GlJIHlNdvyrpFuCw/HodYH+gR0S8mbedCRwHbMNHYas5g/PnBkBE3Fooe0bSGGAXoMXR4OxrwGak0DcHmCPpNOC3ko7On3FzXoqIK/Lr8ZLeAe4tjGheR9PrLecBqwCfkzQ9It4ijU5Wy9YGNgeeiYiXmqij6saIeDS3eRnw83xez5J+aRgVEc/k8hGkz93MzMxaoZ4Rzo3y98mFbRMLr3sBl0iaWf0CvkWasq2aXBNEJhXqbQvda/oE8Ere3hpTC6/nkKZvIU1Lr8ZHI5rN9iMiFpLOtdiPqYXyuY20N7fQXq/8/bnCtX0XWIn6zm0haYT2Q5L2k/RHSe9ImkUKUt3qqKuqOzA9h82qV0hBsJ56pta8n0vT57+IiJhAWrJwGvCWpPskKRePAH5Hmv6eLulqSes1Vk9tPwrnUm13Qwo/7/lnd5HraGZmZvWrZw3nG/l7D1KwgLT2rWoy8JOIaG4ksYekhkLo7MlHo4IL6+vqhxrb/zXS1HFRb9o+JEwnBaLNSNPCjfWjZ/WNpE75fWv7UQ09m0XE9FYcXykGfUndSaO1ewHjI2KepPMAFY5p6fN4DegmadVCYO4N/It0fUoVEZcBl+Xp7yHAbcDGOTSeCpya1xyPJYXQAa1o5g3Szzvw4ch2a395MTMzW+61GDgj4vV8k8W50v9v797DrarKPY5/XxFUJNQSuogIpJaFaDL02EmTLG+lZaV5SVExxcqn0rI6KMZJ826ePMnxDt6OZlYYaqKZPprPk56RlpoXEkEUJVABIU1Fx/njHQvmXqy19lqbtdZmb36f59nPXvM+xpxjzvmuMceYKxwFbACcWpjlAmBSCOHv+OPI9YHRwEulR8t4282Tcsehkfgj4xPztIV4kLMVtdvclcwHBocQBsYYX83jrgcmhhDG4o/pdwDG451jmibGmEIIk/F9MRfvULQZ8O78iH0q8P0Qwr14zeYP8H18axe3tyA/Yp4cQvhOjHFeCGFjvAb5zhjjsk5WUW4AXqu9EHgrhLAz3l7xicI88/FjUc2DwNPA+SGE7+LtJU8DptTxOH21hBB2AtbLaXgDWIq3LSaEsF9O10xgGR4Av93FTV0DnB1C+DV+jL+FdxATERGRLqi309Ch+I3+ObwjzNWlCTHGy/De4VPwXs9zgYn4Y9+S+/Cgcz5wC/AzcvvN3LN5InB9fmx8cidpuRu4E5id598txjgbr+E8Hu+5fA0wMcZ4Y535a8TJwI3ANDzguQcodZA6Fw9+78A7Se0O7FkIjLviGOAp4J4QwlLgUbzDUsPBXW4n+SPgZmAx8ENWti0tOR04LISwKITwuwrrWA7sizeJmIsHfw/gve1bbQBedl7Cj/OewEF52geB6cCreLD/Oh7wd8XVwEXAbfhxHAL8CQ9yRUREpEGWUksrpUR6vNw04jngpBhjzY5u06dPT/vtt197EiYiItL9rJ6Zuv2XhkTWRCGEg/Fa7HXwd3L2B1ap8RUREZHOrXEBZwhhV6rf2M+IMZ7R4Pom4D2bK9knxnhfI+tbkxRefl/uvhjjPm1NDM0/dt3seFa+Kuox4LMxxkXdmB4REZEeS4/URZpIj9RFRGQtU9cjdf2WuoiIiIi0lAJOEREREWkpBZwiIiIi0lIKOEVERESkpRRwioiIiEhLKeAUERERkZZSwCkiIiIiLaWAU0RERERaSgGniIiIiLSUAk4RERERaSkFnCIiIiLSUgo4RURERKSlFHCKiIiISEsp4BQRERGRllLAKSIiIiItpYBTRERERFpKAaeIiIiItJQCThERERFpKUspdXcaRHqN9dZb77E333zzX92djnZZd911N12+fPlL3Z2OdlF+ezflt3dTflvmpZTS3p2mpw0JEVlrbLvttv+KMYbuTke7hBCi8tt7Kb+9m/Lbu61p+dUjdRERERFpKQWcIiIiItJSCjhFmuvS7k5Amym/vZvy27spv73bGpVfdRoSERERkZZSDaeIiIiItJQCThERERFpKb0WSaRBIYStgauA9wAvA2NjjH8vm6cPcCGwN5CAs2KMl7c7rc1QZ373BM4AtgX+O8b4vbYntEnqzO9E4GDgbeAtYEKMcUa709oMdeb3KOAE4B2gD3BZjPHCdqe1GerJb2HeDwEPA5N7apmu8/hOAr4BvJBH3R9j/GY709ks9R7fEMJXgImA4dfoz8QY/9HOtDZDncf3amBUYdQoYP8Y42/bllBUwynSFRcDF8UYtwYuAi6pMM9XgS2BrYCPA5NCCMPalsLmqie/zwBfA85tZ8JapJ78PgjsGGMcBYwDfhFC2KCNaWymevL7K2C7GOP2wL8D3w0hjKowX09QT35LXxovAaa1MW2tUFd+gatjjNvnvx4ZbGad5jeEEIBJwB4xxpHALsCSdiayiTrNb4xxbOnYAkcAi4C2f0FWwCnSgBDCYGAH4Po86npghxDCoLJZD8Jrgd6JMS7Eb1oHti+lzVFvfmOMT8cY/wIsb3MSm6qB/M6IMb6WBx/Ba0ne07aENkkD+X01xljqYdof6IvXCvUoDZy/AD8EbgFmtil5Tddgfnu8BvJ7AnBejHE+QIxxSYyxx/1CXBeP79HAdTHGN1qdvnIKOEUaszkwL8b4NkD+/0IeXzQUeLYwPLfCPD1BvfntLbqS37HArBjj821IX7PVnd8QwudDCH/Dy/W5McZH25rS5qgrvyGE7YC9gAvansLmaqQ8HxxCeCSEcEcI4ePtTGQT1ZvfjwAjQgj3hhAeCiGcEkKwNqe1GRq6XoUQ+gGHAle2LYUFCjhFRLoohLAbcBpwSHenpdVijL+NMX4U2Bo4PLdv7HVCCH3x9xceV7qRrwUuBobnJiLnAjeHEHpcjX0D+uDtGPcAdgP2AQ7v1hS1x/7A3Pw0qu0UcIo05jlgs9y+q9TO6wN5fNFcYIvC8NAK8/QE9ea3t6g7v7kW6Fq88f1TbU1l8zR8fGOMc/E2rPu2JYXNVU9+3w98ELgthDAH+A5wTAhhjXqJdp3qOr4xxvkxxrfy5zvz9JFtTmszNHJ9vinG+EaMcSlwM7BTW1PaHI2ev+PoptpNUMAp0pAY4wLgL6ys0ToEeDi30yz6JX6TWie3p9kfuKl9KW2OBvLbK9Sb3xDCjsAvgANijA+1N5XN00B+tyl83hT4FNDjHqnXk98Y49wY46YxxmExxmHAf+HtsY9te4JXUwPHd7PC5+2BYUCP+xLVwPXqf4E9QwiWa7Q/Dfy1fSltjkauzyGEIcCuwHXtS2FHei2SSOOOA64KIZyK9/YbCxBCuA04NcYYgWuAfwNKr6f4cYxxdncktgk6zW8IYRfgBmAgYCGEg4Gje+irguo5vpOBDYBLvMMrAIf30HaN9eT32Pzqq7fwDlI/jzHe0V0JXk315Lc3qSe/Z4QQRuOv+XoTL8vzuyvBq6me/N4ABOBx/FVfM4Aruie5q63e8nwEMD3GuKh7kqmfthQRERGRFtMjdRERERFpKQWcIiIiItJSCjhFREREpKUUcIqIiIhISyngFBEREZGWUsApshYzs73M7L7C8Bgzm9ONSWobM5tqZpc3cX3DzCwVhgeZ2bNmtmkdyx5nZtc0Ky09gZntamaLuzsdayMzO6yR87zZ54rU1qpzowvH/SwzO61Z21fAKbKWMjPDfyv6R53M93Uze8zMXjWzRWYWzeygwvQ5ZnZYheVWGW9uZl7XgLJpY8wsmdmy/PeCmU0xs3evXk67R0ppIf6C6c7274bAj4FJbUjWGiOldF9KaePuTkc1ZjbJzH7f3elYG7RqX5vZPWZ2SrPX22rl50Y3lsWzgW+a2WadzlkHBZwia689gX7A3dVmMLND8IDpaGAj/GfTTsBfMNwVnwJG4C9brvT742+nlAaklAYAuwAfx3/ppae6EilgX04AAAzQSURBVDjKzAbWmOcw4NGU0qw2pakDM+tjZroXiEgHKaVFwO+A8c1Yny4yIm2Qa/tOMbO7c+3do2Y2yswOMbOnzWyJmV1uZusWlhlqZjeZ2Xwze9HMLjWzdxWmn2Fmz+T1zTKz7xSmDcu1hYeb2eNmttTM7jCz9xeStT/w+1T71x/+Hbg3pfRAcq/nb99d/ZWZ8cDt+C8x1byIpZSeAW4BPlY+zczWzftk/7LxU81sSv78aTN7INfKLjSzG8xscLXt5f21S2F4jJktL9vmhFxDu9jM7jezUHltK/Lwd+Al4DM1ZtsfuLMsLd82syfzcZtrZmeaWZ887Vwzm1Y2/5g874Z5eKSZzcj5Li3fN08rlY2jzexx4DVgsJkdbGZ/zbXPL5rZJaX15eXeZ2bTc1mdmZdPZjasMM8xuTZ8iZk9bGZ7Vst0hf071cyuMbMr8/6dl8+P7c3s/3L+7jazDxSWmWNmp5rZH/N5EM1sx8L0mmXAzPrmY/pUXv8sMzvAvAZ/AjDGVta4j6iSj93yNpbkYza+MG2MmS03s4PyupeY2Y3F87jC+rpyrRhlZn/I+XwmL9+nMH2nvG+Wmdkf8S99xW32N7PzzGy2mb1iZreb2ZbV0lghze8xs6vNr1XzzewqKzyZsLKnHYUyOKTavjazI3N+f5DL4wIzO79COR5SWO+RZvZ0/vxz/KccJ+Z1VvypTvPaw7vM7OxcRl42sxPNbIu8T5ea2Z/NbJvCMqt1rhTK+mWFsr5Kucmfa+6fsrx0aPrQpON+J36NWn0pJf3pT38t/gPm4D9zuQ3QF7gWmAVcCmwIDAUWAF/N868PPI0/at0A2AS4DbiysM7D8BpHA3YHXgf2ytOGAQkP2DbFf3LyfuCywvIPAN8qS+cYYE5h+EDgX8Dp+O8Nb1wlb4d1Nh4YBLwBfAkPIhMwumzbywvDW+K/53xllX16DjCtMDwAWAbsmod3AXbEf8L3fcC9wPWF+acClxeGE7BLjfT8JO+zEUAfvNb3JWCT4j6vkM7pwOk1ysY/gM+XjfsyMDwf24/lecbnaR/Bf35wUGH+q4Ar8ufBwMt4QN8P2AyIwKllZeOuvF/65fzsA3wUr4jYEv/ZvzML27gL+FUuS4OBe/J6huXpx+Bldru8js/m47FllXyX79+peBn+XF7+uLz8b4EhQH/gD3Qsw3OAF4DROR8/BBYCA+ssA2fnfI7K+3oIMCpPm4R/Iat1Xg/PaT4yb2Nn4BXgwEIeE/6ziQOA9+LXgZObeK3YKJePicB6eblngJMK01/O+6Zf3h/z6XieX4dfK96b5/lP4Emgb6VzpUKab8fL+Sb571bg1hrXgmF5vwyptq/zPn0LuAi/Bn4QmAlMqLSOwjJPF4bvAU7p5BhOytv5GivPg7eB35cdgzsLy6zuuTIVLzefz+v4Uk7DFlXOjWr75+mycSuOUzOOe55nNP5Eql+t/VjPX1tvuvrT39r6ly+4JxWGP5svQMWg4Ubggvz5AGBW2TpG4wFbnyrbuAk4J38uXYx3LEz/JvBwYXgmcGTZOsYUL0h53L7Ar/Gb2tv4I/iRZXn7J7C47O8dOt5kvo/fKEs3sYeAS8q2nfKyi4DZwMVUCHLz/NvggdfgPDwOmFnjGOwLLCgMr7g45+GqAScejCwFPlm2zkdLeaR6wHkdMLlGut4ExnRSfs4DbiwMPwCckD+/K+//T+Th7wF/KFv+y+SbU6FsfLKTbR4PPJg/D8nLjChM/zQdb6KPAWPL1jGdKjd8KgecxSClf17/gYVx36BjGZ4DnFYYNmAucGhnZSDPuwz4XJV5J9F5wDkBuL9s3JnAjLIyXTzPzwV+U2Odc2jsWnEo8Bz5p6rzuPHAU/nzV/M+KU7/Cfk8x7+QJmBoYfo6wBLy+UCNgBP/0puArQrjPpTHvb+Qp64EnG8A/QvjvkY+x8vXUVimKwHn38rGLahwDBY18VyZSqGs53ELgS9UOTeq7Z9aAedqH/c8bqs83+Ba+7GevxVV8iLSci8WPr+Gt1dcWDau9KhtODDUVu2pmPCamnlm9i28VmkIfvPcAO+kUm2b/yysHzyoq9W20DeY0i34t2DM7MPAZOAWMxue8hUJr327tricFXpDmpnltF6bUnorj74COMvMvpdSWprHvZ3q7EiSUnrCzB7Ca3p/ChwFTClsczRwBl7j1h/fRwMqrKoem+Zlp1uhJzpe+zGk8iIrDMSD52pWOQ7mbWdPxGtT18VrH/5UmGUK8HW809dXgOdTSvfnacOBT5SVHcNrb4rmlG1zD+BU4MN4TVkf/MYLXksKfgMrebZsfcOBi8zswsK4dYHnqd+K8ppSes2LzSrnTfnj6DmFZZKZzSUfk07KwCC8xnBmA+krtzmrHttZwBcKw+Xnefl5WEkj14rNgWcL52IpDZvnz0MqTC+meXj+/0je3yV9C+uopTRPcZ2zCtNepOsWpJReKwzPofPzrSvK0/gaNcpdE86VStusp1w0olnHfSArKwJWi9pwiqyZnsW/yW9c9rd+SmmemX0Cfxw4Htg0B2nT8RtqvR7GH8/WLaX0JB7kbIE/OqvX7vijp3Gldl7445sBeA1NV00BjsztjnYGri5MuwGvRd06pTSQyp2UipbhAUjJBwqfX8JvCJ8pOx4bppTO6mS9I/F9XU2H42Bmm+OP8E7Ha4g2wh8rFo/tDcDWZrYDXtMxpTDtWbw2pJjOjZJ3xCp6p7DNfsC0vN6heX/9oLDNefn/0MLyxc+l7Y4r2+6AlNLXa+S9GYaVPuQvNkNZGeTWKgML8UBiqyrrfafK+KLnitvPRuTx7fIcsIV1jBqKaZhXYfqwwudSMLRV2bHrn1K6vs7tl69zRNm0pVQ/t6D6vh5sZv3L0l06tqUvqV1Zb5c16VxpVKV8lO9T6Jj/Zh33kXgN8JtdTPsKCjhF1ky3AP3MOzS8y9xmZvbFPH0g/nh7IZDM7HN4u6JGTMMf9VRlZuPM7EDL75LMDfSPAx5PKb3SwLbG4+3nPgxsn/9G4oHSsQ2mu+gGPJC9EG9jNa8wbSD+eGipmQ3F2zLV8mfgCDPrlxv3n1iakGsJfgacZ2ZbAZjZAPP3mJbf5FbIgfAgvD1YNdPo2KloAH5tXgi8ZWY7A4cXF0gpLQZ+gwelO+NtOEuuBkI+duub2Tq5k8HeNdLQD6+pWZRSet3MPoI/Jixt73n88eRZuTwOAspfN3MBMMm8k4+Z2QZmtkuuFW+lcWa2g3lnkpPwmsxb87SqZSAf08nAOeadrMy8E8uoPMt8/ClDvxrbvh4YbWZjzTuV7YSX9SuamsPabsWP3YRcdj+EB0ClNNyCl6mTzDtJ7YC3PwYgpbQAfzIy2fLrb8xsYzP7opW9uqySlNILwB3A+Xm5TYDzgd+llEq1eH8GDsnnzCC8vWlRtX29DnB2Lksj8OYiV+Xtvkz+kmP+poVt8aco5eutu/NTnZpxrjSq0v75Cx6Q75vP8S8CnyxMb9Zx3wO/Rq02BZwia6D8GGl3vObrSfymeRceqAHMwAOLB/HatwPwAKQRM4DlZjamxjyL8Ee3T5jZP/G2g4vxtnB1Me8VvD9wXkppfvEPr6X9mHXS27ualNISPN/74K8gKjoWb/O1FG+D+stOVnc8fnN6BW8jN7Vs+o+Am4GbzexVvGPHcdS+jo4DpuZ0VnMNsF2+oZJSeqKwrcV4kFSppmkKnu8ZhRs7eb9+Ct/nc/Bj+BvKeqgWpZSW4cf5HDNbhteoljfPOBQP5p7HO6CV9ucbeR2X4R25puRtzsUDi7418t4Ml+JfOBYBB+FtMkv7u7MycDJ+rKflee5hZYDyS7yGbr55T+LhZcuSUpqNt+87Hu+gcQ0wMaV0Y7My15mc1z3xLy3/YOW14ad5+mK8I9ZB+D66EPifstUcg3fQu8fMluJtkw/EH6XW4zB8/z2FX68WA2ML00/BvyC/iO/jG8qWr7avn8XL22z82nM7XsZKjsCvRUtyfssD/QvwL1+LzexvdealpmacK12wyv5J/hq1b+Pl/xVgb7yjUimdq33czWxjvHxf3MV0d2AdH++LyNok13pNSCl9Mg+PwQOkYd2Zrp4o14rOTilZHh6E9w4PZe3vKi17HN7p5/Ba861JzGwvPCjeIHXTjcS8nfAp5e2HpeczsyPxY9vsGsq2WxPOla4wszPx9sNNeXm+Og2JrMVSSrfjtQbSZDnI3KLOeS+mSbUIrWJm2+NtyR7FOxycDvyiJ91ARdqht5wrKaX/aOb69EhdRIrm0LN/2ac7LcY7QvVWm+CPpZcBfwQewR/piUhHOlcq0CN1EREREWkp1XCKiIiISEsp4BQRERGRllLAKSIiIiItpYBTRERERFpKAaeIiIiItNT/A50EGL6Zi54EAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x396 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "shap.summary_plot(shap_values_xgb, X, plot_type=\"bar\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/navdeep/article-information-2019/models/XGB_BEST_GRID_v2_HMDA_1575928848.548365/xgb_grid2_model_11\n"
     ]
    }
   ],
   "source": [
    "# Save the model\n",
    "model_path = h2o.save_model(model=best_xgb, path=\"../models/XGB_BEST_GRID_v2_HMDA_\" + str(time.time()), \n",
    "                            force=True)\n",
    "\n",
    "print(model_path)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "information-article",
   "language": "python",
   "name": "information-article"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}