d_final_models_VPAS_RDkit.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Initialisations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-17T13:23:37.612946Z",
     "start_time": "2020-01-17T13:23:34.306162Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd ; pd.set_option('display.max_columns', 500) # dataframes\n",
    "import numpy as np # mathsy bits\n",
    "import ipywidgets as widgets # widgets\n",
    "\n",
    "from sklearn.model_selection import train_test_split #split data into train and test sets\n",
    "\n",
    "# feature selection + gridsearch\n",
    "from sklearn.feature_selection import SelectFromModel, SelectKBest\n",
    "from sklearn.feature_selection import f_regression, mutual_info_regression\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "# models\n",
    "from sklearn.linear_model import Lars, ElasticNet, Lasso\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "from sklearn.ensemble import RandomForestRegressor, ExtraTreesRegressor\n",
    "from sklearn.ensemble import AdaBoostRegressor, GradientBoostingRegressor\n",
    "\n",
    "import joblib # saving models\n",
    "from datetime import datetime # get time for labelling saved models\n",
    "\n",
    "# visualization\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "import mpld3 # hover-over labels for plots\n",
    "mpld3.disable_notebook()\n",
    "\n",
    "train_scores = dict() ; test_scores  = dict() #to hold scores for viewing at the end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-17T13:30:15.556493Z",
     "start_time": "2020-01-17T13:30:15.536439Z"
    }
   },
   "outputs": [],
   "source": [
    "def grid_search_return_hp_dict(model, X_train_gs, y_train_gs, param_grid):\n",
    "    \"\"\"Run GridSearchCV on given model with defined param_grid, return best params as dict.\n",
    "    Pre-assign sklearn model to variable. Pass param_grid as dict.\"\"\"\n",
    "    print(str(model).split(sep='(')[0], \"grid search.\")\n",
    "    GS = GridSearchCV(model, param_grid, n_jobs=2)\n",
    "    GS.fit(X_train_gs, y_train_gs)\n",
    "    print(GS.best_params_)\n",
    "    return(GS.best_params_)\n",
    "\n",
    "def produce_exp_vs_pred_df(features_list, model, codename):\n",
    "    \"\"\"Pass features_list as a string\"\"\"\n",
    "    pred_list = []\n",
    "    for i in range(0, len(rdkit_features)):\n",
    "        prediction = model.predict([rdkit_features[eval(features_list)].iloc[i]])\n",
    "        pred_list.append(prediction)\n",
    "    train_scores.update({codename : model.score(X_train, y_train)})\n",
    "    test_scores.update({codename : model.score(X_test, y_test)})\n",
    "    \n",
    "    print('Training Score:\\t', model.score(X_train, y_train))\n",
    "    print('Testing Score:\\t', model.score(X_test, y_test))\n",
    "    \n",
    "    exp_vs_calc = pd.DataFrame(constants_first)\n",
    "    exp_vs_calc['Predicted'] = pred_list\n",
    "    exp_vs_calc.rename({'Kh_first':'Experimental'}, inplace=True, axis=1)\n",
    "    \n",
    "    now = datetime.now()\n",
    "\n",
    "    dt_string = now.strftime(\"_%d_%m_%Y_%H_%M_%S\")\n",
    "    filename = \"models/RDKit_VPAS/\" + str(model).split(sep='(')[0] + \"_\" + features_list[16:] + dt_string + \".joblib\"\n",
    "    print(str(model).split(sep='(')[0], \"run at:\", now, \". Saving to\", filename)\n",
    "    \n",
    "    joblib.dump(model, filename)\n",
    "    return exp_vs_calc\n",
    "\n",
    "def prediction_plot_scores(model_func, pred_df):\n",
    "    \"\"\"Print train+test scores, then plot scatter of predicted vs actual HLCs. Uses {X/y}_{train/test},\n",
    "    redefining these variables will change the output.\"\"\"\n",
    "    #print('Training Score:\\t', model_func.score(X_train, y_train))\n",
    "    #print('Testing Score:\\t', model_func.score(X_test, y_test))\n",
    "    fig = plt.figure(figsize=(10, 10))\n",
    "    scatter = plt.scatter(pred_df['Experimental'], pred_df['Predicted'])\n",
    "    plt.xlabel('Experimental')\n",
    "    plt.ylabel('Predicted')\n",
    "    plt.title('%s predictions of HLCs' %str(model_func).split(sep='(')[0])\n",
    "    labels = ['{}'.format(i) for i in species_names]\n",
    "    tooltip = mpld3.plugins.PointLabelTooltip(scatter, labels=labels)\n",
    "    mpld3.plugins.connect(fig, tooltip)\n",
    "    plt.plot([-30, 10], [-30, 10], c='red')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DRAGON descriptors\n",
    "## Bringing in data and splitting it into parts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-17T13:23:57.469717Z",
     "start_time": "2020-01-17T13:23:57.400534Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input Shape (4020, 27)\n",
      "Removed NaN, new shape (4013, 27)\n"
     ]
    }
   ],
   "source": [
    "csv = pd.read_csv('organics_RDKit.csv') # contains VP/AS HLCs\n",
    "\n",
    "print(\"Input Shape\", csv.shape) #input shape\n",
    "\n",
    "csv.dropna(axis=0, inplace=True)\n",
    "\n",
    "print(\"Removed NaN, new shape\", csv.shape) #removed NaN shape\n",
    "\n",
    "smiles_strings = csv.pop('SMILES')\n",
    "species_names = csv.pop('Species')\n",
    "constants_first = csv.pop('Constant')\n",
    "\n",
    "\n",
    "rdkit_features = csv.drop(['SMARTS', 'CAS'], axis=1) #seperating features\n",
    "\n",
    "X_train_main, X_test_main, y_train, y_test = train_test_split(rdkit_features, constants_first, test_size=0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-17T13:25:17.267782Z",
     "start_time": "2020-01-17T13:25:17.232689Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>wt</th>\n",
       "      <th>logp</th>\n",
       "      <th>fraction_sp3</th>\n",
       "      <th>ali_homo_rings</th>\n",
       "      <th>ali_hetero_rings</th>\n",
       "      <th>amide_bonds</th>\n",
       "      <th>aro_homo_rings</th>\n",
       "      <th>aro_hetero_rings</th>\n",
       "      <th>hba</th>\n",
       "      <th>hbd</th>\n",
       "      <th>volume</th>\n",
       "      <th>tpsa</th>\n",
       "      <th>asph</th>\n",
       "      <th>ecce</th>\n",
       "      <th>isf</th>\n",
       "      <th>npr1</th>\n",
       "      <th>npr2</th>\n",
       "      <th>pmi1</th>\n",
       "      <th>pmi2</th>\n",
       "      <th>pmi3</th>\n",
       "      <th>rog</th>\n",
       "      <th>si</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2756</th>\n",
       "      <td>248.000700</td>\n",
       "      <td>3.2370</td>\n",
       "      <td>0.300000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>240.608</td>\n",
       "      <td>46.53</td>\n",
       "      <td>0.487798</td>\n",
       "      <td>0.976286</td>\n",
       "      <td>0.001542</td>\n",
       "      <td>0.216484</td>\n",
       "      <td>0.909603</td>\n",
       "      <td>589.795173</td>\n",
       "      <td>2478.152795</td>\n",
       "      <td>2724.433466</td>\n",
       "      <td>3.409831</td>\n",
       "      <td>2.580028e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1709</th>\n",
       "      <td>138.079313</td>\n",
       "      <td>0.8596</td>\n",
       "      <td>0.142857</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>158.896</td>\n",
       "      <td>61.27</td>\n",
       "      <td>0.273154</td>\n",
       "      <td>0.915096</td>\n",
       "      <td>0.002190</td>\n",
       "      <td>0.403235</td>\n",
       "      <td>0.607198</td>\n",
       "      <td>277.212404</td>\n",
       "      <td>417.431304</td>\n",
       "      <td>687.471164</td>\n",
       "      <td>2.236404</td>\n",
       "      <td>7.030994e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2117</th>\n",
       "      <td>247.063329</td>\n",
       "      <td>4.4922</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>250.480</td>\n",
       "      <td>43.14</td>\n",
       "      <td>0.412458</td>\n",
       "      <td>0.964075</td>\n",
       "      <td>0.001256</td>\n",
       "      <td>0.265630</td>\n",
       "      <td>0.746090</td>\n",
       "      <td>594.133388</td>\n",
       "      <td>1668.779150</td>\n",
       "      <td>2236.699126</td>\n",
       "      <td>3.016489</td>\n",
       "      <td>8.284874e-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3411</th>\n",
       "      <td>623.762951</td>\n",
       "      <td>3.4399</td>\n",
       "      <td>0.466667</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>425.600</td>\n",
       "      <td>102.29</td>\n",
       "      <td>0.219924</td>\n",
       "      <td>0.897871</td>\n",
       "      <td>0.000141</td>\n",
       "      <td>0.440259</td>\n",
       "      <td>0.649133</td>\n",
       "      <td>4617.761483</td>\n",
       "      <td>6808.589966</td>\n",
       "      <td>10488.743030</td>\n",
       "      <td>4.177445</td>\n",
       "      <td>1.937191e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2974</th>\n",
       "      <td>387.818595</td>\n",
       "      <td>7.5050</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>308.296</td>\n",
       "      <td>18.46</td>\n",
       "      <td>0.527459</td>\n",
       "      <td>0.980627</td>\n",
       "      <td>0.000646</td>\n",
       "      <td>0.195884</td>\n",
       "      <td>0.804116</td>\n",
       "      <td>1244.454303</td>\n",
       "      <td>5108.555795</td>\n",
       "      <td>6353.010098</td>\n",
       "      <td>4.031595</td>\n",
       "      <td>7.770000e-12</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              wt    logp  fraction_sp3  ali_homo_rings  ali_hetero_rings  \\\n",
       "2756  248.000700  3.2370      0.300000               0                 0   \n",
       "1709  138.079313  0.8596      0.142857               0                 0   \n",
       "2117  247.063329  4.4922      0.000000               0                 0   \n",
       "3411  623.762951  3.4399      0.466667               0                 0   \n",
       "2974  387.818595  7.5050      0.000000               0                 1   \n",
       "\n",
       "      amide_bonds  aro_homo_rings  aro_hetero_rings  hba  hbd   volume  \\\n",
       "2756            0               1                 0    3    1  240.608   \n",
       "1709            0               1                 0    3    2  158.896   \n",
       "2117            0               4                 0    2    0  250.480   \n",
       "3411            0               1                 0    7    2  425.600   \n",
       "2974            0               2                 0    2    0  308.296   \n",
       "\n",
       "        tpsa      asph      ecce       isf      npr1      npr2         pmi1  \\\n",
       "2756   46.53  0.487798  0.976286  0.001542  0.216484  0.909603   589.795173   \n",
       "1709   61.27  0.273154  0.915096  0.002190  0.403235  0.607198   277.212404   \n",
       "2117   43.14  0.412458  0.964075  0.001256  0.265630  0.746090   594.133388   \n",
       "3411  102.29  0.219924  0.897871  0.000141  0.440259  0.649133  4617.761483   \n",
       "2974   18.46  0.527459  0.980627  0.000646  0.195884  0.804116  1244.454303   \n",
       "\n",
       "             pmi2          pmi3       rog            si  \n",
       "2756  2478.152795   2724.433466  3.409831  2.580028e-01  \n",
       "1709   417.431304    687.471164  2.236404  7.030994e-02  \n",
       "2117  1668.779150   2236.699126  3.016489  8.284874e-03  \n",
       "3411  6808.589966  10488.743030  4.177445  1.937191e-01  \n",
       "2974  5108.555795   6353.010098  4.031595  7.770000e-12  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test_main.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature Selection\n",
    "Made aware of features selection methods in scikitlearn via https://towardsdatascience.com/feature-selection-techniques-in-machine-learning-with-python-f24e7da3f36e\n",
    "\n",
    "See also [scikit-learn docs](https://scikit-learn.org/stable/modules/feature_selection.html)\n",
    "### Importances via 'dummy' GradientBoostingRegressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-17T13:31:09.072713Z",
     "start_time": "2020-01-17T13:31:08.591436Z"
    }
   },
   "outputs": [],
   "source": [
    "fs_model = GradientBoostingRegressor()\n",
    "fs_model.fit(X_train_main, y_train)\n",
    "importances = pd.Series(fs_model.feature_importances_, index = X_train_main.columns)\n",
    "top_15_features_model = list(importances.nlargest(8).index.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-17T13:31:10.101448Z",
     "start_time": "2020-01-17T13:31:10.096435Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['volume', 'asph', 'npr1', 'fraction_sp3', 'wt', 'npr2', 'hbd', 'ecce']"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_15_features_model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Importances via 'dummy' LASSO model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-17T13:31:15.391506Z",
     "start_time": "2020-01-17T13:31:14.473066Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\coordinate_descent.py:475: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Duality gap: 1.2632506705927678e+64, tolerance: 2.888971159359407e+60\n",
      "  positive)\n"
     ]
    }
   ],
   "source": [
    "dummy_lasso = Lasso(max_iter=10000, alpha=0.5)\n",
    "selector = SelectFromModel(dummy_lasso).fit(X_train_main, y_train)\n",
    "coefs = selector.estimator_.coef_\n",
    "importances_lasso = pd.Series(coefs, index = X_train_main.columns)\n",
    "top_15_features_lasso = list(importances_lasso.nlargest(8).index.values)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SelectKBest using f_regression scoring function\n",
    "F-test estimates degree of linear dependency"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-17T13:31:19.576630Z",
     "start_time": "2020-01-17T13:31:19.565599Z"
    }
   },
   "outputs": [],
   "source": [
    "fs_kbest = SelectKBest(score_func= f_regression, k=15)\n",
    "fit = fs_kbest.fit(X_train_main, y_train)\n",
    "scores_f = pd.Series(fit.scores_, index=X_train_main.columns)\n",
    "top_15_features_kbest = list(scores_f.nlargest(8).index.values)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SelectKBest using mutual_info_regression\n",
    "Catch any statistical dependancy, but requires more samples than f_reg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-17T13:31:27.034447Z",
     "start_time": "2020-01-17T13:31:24.239020Z"
    }
   },
   "outputs": [],
   "source": [
    "fs_mut_info = SelectKBest(score_func= mutual_info_regression, k=15)\n",
    "fit = fs_mut_info.fit(X_train_main, y_train)\n",
    "scores_mutual = pd.Series(fit.scores_, index=X_train_main.columns)\n",
    "top_15_features_mutual_info = list(scores_mutual.nlargest(8).index.values)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature Selection results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-17T13:32:01.545162Z",
     "start_time": "2020-01-17T13:32:00.758071Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA5cAAAJcCAYAAABtzVAmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzdebhdZX33//fHBJlHwRpQG6sIGoKRUREBRVCLKEos0MhQbSn1UZ4OKihi0aLF6anyQ4poI7ZVhEYFKg6IEhEIYAKBAGotErQEKihEkIAM398fax3YOTkZV072Gd6v68p19l7TvvfJ/u61Puu+1zqpKiRJkiRJ6uIp/W6AJEmSJGn0M1xKkiRJkjozXEqSJEmSOjNcSpIkSZI6M1xKkiRJkjozXEqSJEmSOjNcSoMkmZ3kz1dx2UryvOFukzQgyeT2czdxLWzrD5JcnuT+JJ9cG+1bm5K8PMlP+90OaSRKcnOS/frdDknqZbgcZ5IsTLKkPZi8L8lVSY5LMuo+C0lOaQ+yjx80/a/b6af0qWkaI9p6+X2SrQdNn99+xiav4nbW2UmIts2vWsXFjwXuATarqr8bxmatksG/p6r6YVXt0M82aXTp2cc90PNv2yGW2y/J4+38+5P8NMmf9aPNa6qqplTV7H63Q+pqRfutJM9pa/XMIea9od0f/zbJPUm+N7BfTrJFkplJ7mpr/L+SnNCzbpK8O8nP2u+MXyQ5Lcn6w/U+x4tRFyi0VhxcVZsCfwicBpwA/Et/m7TG/gs4etC0o9rp0tpwG3DEwJMkU4EN+9ecteoPgVuqqlZ3xbXRcyoNk4OrapOef4uWs9yiqtoE2Az4G+BzSdb6yQxrRerkKOBe4PDe4NeeiPxX4O+AzYHnAGcCj7eL/BOwCfCCdv7rgVt7tns6zQnWo4BNgdcCrwTOH8b3Mi4YLsexqlpcVRcBhwFHJ9kJlh0WmuSYJFf0PK8kb2/P9tyf5B+SPDfJnPbs0flJntouu1+S/0nyniS/SnJnkkOS/HF7Fuk3Sd7XLvuMJA8meVrPa+2a5O4k6y3nbfwI2CjJlHb5KTQH/j/qXSjJXyT57/b1Luo9k53kgCQ/SbI4yRlABq371iQ/TnJvku8k+cM1+HVr9Po3mp3PgKNpdmhPWFHNJLm8nXxD20ty2OCaapd7otcuyUFJrm/r6Zdr2gs/8DpJPtF+fm9L8tp23jnte3lP265XJVk/yaeSLGr/fWpgZ95TyyckuQv4wurUd7uNPdrvifvaZc/o+a4Y6ve0X5L/6Vn/Be3v+r40QwJf3zPvnCSfSXJx+710TZLnrsnvTeNLNb4J/AbYeWB6kh2TfLf9HP80yZ/0zHtakv9sa/RHSU4dYj/5f5L8DPjZKmzvj5Pc0n5270jyrnb61km+0X7mf5Pkh2lHGqWnt2cVa/fveup0VPXSalw7Cng/8AhwcM/0acBtVfW9tobvr6qvVtUv2vm7A1+uqnur6vGq+klVzQJIsj3wdmBGVc2pqker6mbgUOA1SV65zt7dGGS4FFV1LfA/wMtXY7XXALsCLwHeA5wNzACeBexET08P8AxgA2A74APA54C3tOu/HPhAkj+qqruA2cCf9Kz7FuArVfXICtrSe/A/1IH/K4F/bLc7Cbgd+Eo7b2vgqzRfXFvTnNV6Wc+6hwDvA94EbAP8EDh3BW3R2HM1sFkbbCbQnIz591Vduar2aR++qO1FOW8VVvsdzWd6C+Ag4K/az+Ka2BP4Kc3n+2PAvyRJVR0DfAn4WNuuS4GTaGp6GvAiYA+a2hjwDGArmh7PY3umrbS+22Ufo+kh2hp4KbA/zQ5+pb+nNCeY/hO4BHg68E7gS1m6p+kI4IPAlsB/Ax9ezd+VxqEkT2lPVGxN87khycbAd4Ev03zejgDOTHsiE/gMTZ0+g2a/M3gEDcAhNPX3wlXY3r8Af9mOKtoJ+H47/e9o9s/bAH9Asz8aaqTBqtTu5jR1+jbgM0m2XIVfj9Q3SV4OPJPmmO18lj7Rex2wY5J/SvKKJJsMWv1q4MNJ/qwNk732B/6nPf59QlX9sl3vgLX5PsYbw6UGLKI5aFxVH62q37Znem4CLqmqn1fVYuBbwIt7ln0E+HAbEL9CswP/dHuW6WbgZp48W/xFmgNT2gP5I2jC44r8O3BEe/B5OMse+M8AZlbVdVX1MPBe4KVpxuX/Mc2wwFlt+z4F3NWz7l8C/1hVP66qR4GPANNi7+V4M3AC4wDgJ8Adw/liVTW7qha0Z1tvpDmhse8abu72qvpcVT1GU1+TaA5ShzID+FBV/aqq7qYJakf2zH8c+PuqeriqlrTTVrm+q2peVV3dniVeCHx2Nd7XS2iGOJ1WVb+vqu8D32DpE1lfq6pr21r9Es2BtsafC9qevvuSXLCC5bZNch+wBPg68LdVdX0773XAwqr6Qvt5vY7mROT0dt90KE0tPFhVt9DU1mD/WFW/aWtludtrl32EJoRu1va0XNczfRLwh1X1SHsd8lDhcmW1+0g7/5G2l/YBwOuZNdIdDXyrqu6lOTHz2iRPB6iqnwP70ZwwOR+4px3BMhAy30mzH3gHcEua0WuvbedtDdy5nNe8s52vNWS41IDtaIYErar/7Xm8ZIjnvWeQft0e2A7MG2r9geUvpNnB/hHNgfziwWeWBmuHQPw3TfD7WXvmqde2NL2VA8s/APya5j1vC/yyZ171Pqfpofn0wIEKze8o7boaP/4N+FPgGAb1jA+HJHsmuSzNkPDFwHGs+c7uiZMlVfVg+3DwGd4BS9VK+7j3Zih3V9VDg9ZZ5fpO8vx2iN9dSX5LU7Or+r62BX5ZVY/3TLudpWux98TQgyz/fWpsO6Sqtmj/rajHf1FVbUFzzeXpNNdbDfhDYM+ekHofTYB7Bk0v4kSW3lcM3u8Mnrai7UETVv8YuD3JD5K8tJ3+cZr92yVJfp7kxOW8l5XV7q/bky4DrA+NaEk2BN5MExCpqjnAL2j2xbTTrq6qP6mqbWhGyuxD04tPVS2pqo9U1a7A02gC6H8k2YrmRnaTlvPSk9r5WkOGS5Fkd5oDtIHrRX4HbNSzyDOWWWmYtAeu59PsdI9k5b2WAwYu6h7qwH8RzY4deGK409Noep/upBnKOzAvvc9pDg7+sudAZYuq2rCqrlr1d6XRrqpup7mxzx8DXxtikdWtmaWWTzJ4+S8DFwHPqqrNgbMYdC3wMFmqVoBnt9MGrPaNfwb5Z5qe3+2rajOaIX6r+r4WAc/K0ne2fjbD3Iussa8d0XICMLVn+PkvgR8M+u7fpKr+CrgbeJRmuN6AZ7Gs3npZ0faoqh9V1RtohsxeQHtTkXYEwN9V1R/RXG/2t0n2H+K1Vla70mjzRpoTP2e2JyTvojlWPWqohavqRzT7552GmDdwMnNjmhv/fJ9mf7JH73JJnkUzSuZ7a/F9jDuGy3EsyWZJXkczlO3fq2pBO2s+8KYkG6W5wcjb1nHT/pWmh+j1rPq1becBBzL0Xb6+DPxZkmntDQ4+AlzTDsu7GJiS5E1p7uh3PEsHg7OA9+bJGwZtnuTNq/+WNAa8DXhlVf1uiHkrq5n/Bf6o5/kNNJ+7aUk2AE4ZtPymwG+q6qF25/enrBvnAu9Psk17PfIHWI3rS1fBpsBvgQeS7Aj81aD5g39Pva6hCeXvSbJemr/vdzDt9dNSF1X1e+CTNJ95aIZcPz/Jke3nbb0kuyd5QdtT/zXglLbmd2Q5B7w9lru9JE9NMiPJ5u3w8t/SXJ9MktcleV574nNg+mNDbH+4a1cabusl2WDgH81+dCYwleYSh2k098SYlmRqkr3T3Kzx6dDcMIvmuPHq9vnJbY09td3e/wXuA35aVf9Fc3z3pSQvSTKhPc77KnBpew8CrSHD5fj0n0nupzmTehLw/4DeO8f9E/B7mgO9L9IOSVhXqupKmmu7rmsD4Kqss6SqLu25Dqx33veAk2m+NO4EnktzbSZVdQ/NsIvTaIbKbg9c2bPu14GPAl9ph/HdRHO7ao0zVXVrVc1dzuyV1cwpwBfb4XB/0u7YPgRcSnMnySsGLf924ENtnX6AdXdr9FOBucCNwAKaGyacuha3/y6aoHw/zY1/Bt/c6BR6fk+9M9qD/9fT1N89NLecP6qqfrIW26fxbSbw7CQHV9X9NCcsD6fpAbyLZl8w8KcQ3kFzg5y7aEbYnAs8vLwNr8L2jgQWtvuZ42jvPUCzT7qU5hrJOcCZNfTfthzu2pWG2zdpLqMY+Lcv8Kmquqvn3zzg2zTXYt5Hs09YkOSBdvrXaW5cB83IgS/Q7C8W0VxqdVB7aRQ0Nfx5mpMwA+vPphmirg4y9HXhUn8l+T7NLaQ/3++2SJK0Ikk+Cjyjqoa6a6wkjRv2XGrEaa8B3YVlezUkSeq7NH+zcuc09qAZwvf1frdLkvptYr8bIPVK8kWavw32f9thRJIkjTSb0gyF3Rb4Fc31mhf2tUWSNAI4LFaSJEmS1JnDYiVJkiRJnRkuJUmSJEmdec3lath6661r8uTJ/W6GtFbNmzfvnqrapt/t6GWtaSwaabVmnWksss6k4beiOjNcrobJkyczd+7y/sydNDolub3fbRjMWtNYNNJqzTrTWGSdScNvRXXmsFhJkiRJUmeGS0mSJElSZ4ZLSZIkSVJnhktJkiRJUmeGS0mSJElSZ4ZLSZIkSVJnhktJkiRJUmeGS0mSJElSZ4ZLSZIkSVJnhktJkiRJUmeGS0mSJElSZ4ZLSZIkSVJnhktJkiRJUmeGS0mSJElSZ4ZLSZIkSVJnhktJkiRJUmeGS0mSJElSZ4ZLSZIkSVJnhktJkiRJUmeGS0mSJElSZ4ZLSZIkSVJnhktJkiRJUmeGS0mSJElSZxP73YDRZMEdi5l84sUrXGbhaQeto9ZIY9eCOxb3uwnSmLcq+7SRxP2rRiP3Zxpv7LmUJEmSJHVmuJQkSZIkdWa4lCRJkiR1ZriUJEmSJHVmuJQkSZIkdWa4lCRJkiR1ZriUJEmSJHVmuJQkSZIkdWa4lCRJkiR1ZriUJEmSJHVmuJQkSZIkdWa4lCRJkiR1ZriUJEmSJHVmuJQkSZIkdWa4lCRJkiR1ZriUJEmSJHVmuJQkSZIkdWa4lCRJkiR1ZriUJEmSJHVmuJQkSZIkdWa4lCRphEkyOclNQ0yfnWS3VVh/YZKth6d10viRZNsks9rHBySZl2RB+/OV/W6fNNJM7HcDJI1+SQKkqh7vd1skSVpbqmoRML19eg9wcFUtSrIT8B1gu741ThqBRnXPZZK3JLk2yfwkn00yIclrklyX5IYk32uX2yTJF9ozTTcmObSdfmCSOe3y/5Fkk/6+I2n0aHtWfpzkTOA64Mi2xm5K8tGe5d6W5L/aHpfPJTmjf62WRpUJbc3cnOSSJBu209+S5Kq21vYASPK0dpnrk3wWSP+aLY0s7f7qJ0m+2B4HzkqyUdvD/5H2WHBukl2SfCfJrUmO61n3JoCqur4NmwA3AxskWb9f70saiUZtuEzyAuAw4GVVNQ14DHgL8Dng0Kp6EfDmdvGTgcVVNbWqdga+3w4Xej/wqqraBZgL/O26fh/SKLcD8K/AQcA/AK8EpgG7JzkkybY09fcS4ABgx341VBqFtgc+U1VTgPuAQ9vpG1fVXsDbgZnttL8HrqiqFwMXAc9e142VRrgdgLPb48Df0tQPwC+r6qXAD4FzaHopXwJ8aCXbOxS4vqoeHp7mSqPTaB4Wuz+wK/CjZkQeGwJ7ApdX1W0AVfWbdtlXAYcPrFhV9yZ5HfBC4Mp2/acCcwa/SJJjgWMBJmy2zXC9F2m0ur2qrk7yBmB2Vd0NkORLwD7tMj8YqMUk/wE8f6gNWWvSMm6rqvnt43nA5PbxuQBVdXmSzZJsQVNvb2qnX5zk3qE2aJ1pHPtlVV3ZPv534Pj28UXtzwXAJlV1P3B/kofa2lpGkinAR4EDlzPfOtO4NWp7LmmG/Hyxqqa1/3YAPgjUcpYdPD3Ad3vWf2FVvW3wilV1dlXtVlW7Tdho87X+JqRR7nftz+UNwVvloXnWmrSM3h6Rx3jyhPDg/VktZ/oyrDONY8urm4E6e5yla+5xhuiESfJM4OvAUVV165AvZJ1pHBvN4fJ7wPQkTwdIshVwA7Bvkuf0TAO4BHjHwIpJtgSuBl6W5HnttI2SDNmjImmlrqGpva2TTACOAH4AXNtO3zLJRJ4c1idpzR0GkGRvmks+FgOXAzPa6a8Ftuxf86QR6dlJXto+PgK4YnU30PZkXgy8t6cXVFKPURsuq+oWmmsmL0lyI/BdYBLNMISvJbkBOK9d/FRgy/bmBzcAr2iH7x0DnNuufzVeDyatkaq6E3gvcBnNSZ7rqurCqroD+AhN+LwUuAVY3LeGSmPDvUmuAs4CBkbcfBDYJ8l1NEP1ftGvxkkj1I+Bo9tjvq2Af16DbbwDeB5wcnszyfkDnRySGqP5mkuq6jyeDJC9vjVouQeAo4dY//vA7sPTOmlsq6qFwE49z78MfHmIRb9cVWe3PZdfpxlJIGkFhqivT6xk+V+z9PVffzM8LZNGrcer6rhB0yYPPKiqc2hu6DPwfGDePbS1WFWn0nRYSFqOUdtzKWnUOCXJfOAm4Dbggj63R5IkScNgVPdcShr5qupd/W6DJGn8GjwSQNLwsedSkiRJktSZ4VKSJEmS1JnhUpIkSZLUmeFSkiRJktSZ4VKSJEmS1JnhUpIkSZLUmeFSkiRJktSZ4VKSJEmS1JnhUpIkSZLUmeFSkiRJktSZ4VKSJEmS1JnhUpIkSZLUmeFSkiRJktSZ4VKSJEmS1JnhUpIkSZLUmeFSkiRJktSZ4VKSJEmS1JnhUpIkSZLUmeFSkiRJktTZxH43YDSZut3mzD3toH43Qxrzpm63eb+bII157tOk4ef+TOONPZeSJEmSpM4Ml5IkSZKkzgyXkiRJkqTODJeSJEmSpM4Ml5IkSZKkzgyXkiRJkqTODJeSJEmSpM4Ml5IkSZKkzgyXkiRJkqTODJeSJEmSpM4Ml5IkSZKkzib2uwGjyYI7FjP5xIvX+nYXnnbQWt+mNJotuGNxv5sgjXnDtU8badzHqp8G6szPocYLey4lSZIkSZ0ZLiVJkiRJnRkuJUmSJEmdGS4lSZIkSZ0ZLiVJkiRJnRkuJUmSJEmdGS4lSZIkSZ0ZLiVJkiRJnRkuJUmSJEmdGS4lSZIkSZ0ZLiVJkiRJnRkuJUmSJEmdGS4lSZIkSZ0ZLiVJkiRJnRkuJUmSJEmdGS4lSZIkSZ0ZLiVJkiRJnRkuJUmSJEmdGS4lSZIkSZ0ZLiVJkiRJnRkuJUmSJEmdjZlwmWRykpuGmD47yW6rsP7CJFsPT+uk8SXJtklmtY8PSDIvyYL25yv73T5pLEvyvn63QRoplnd8KGl4jJlwKWnkqKpFVTW9fXoPcHBVTQWOBv6tfy2TxgXDpSSpL8ZauJyQ5HNJbk5ySZIN2+lvSXJVkpuS7AGQ5GntMtcn+SyQ/jVbGnnas70/SfLFJDcmmZVko7aX/yNJ5iSZm2SXJN9JcmuS43rWvQmgqq6vqkXtZm8GNkiyfr/elzTaJXlPkuPbx/+U5Pvt4/3bEQMbJpmf5Et9bag0TE444QTOPPPMJ56fcsopfPKTn+Td7343wJR2pMxhg9dLckySM3qefyPJfu3jB5J8tB1hc2mSPdrRbz9P8vp2mQlJPp7kR+1+8S+H+71Ko81YC5fbA5+pqinAfcCh7fSNq2ov4O3AzHba3wNXVNWLgYuAZ6/rxkqjwA7A2VW1M/BbmhoC+GVVvRT4IXAOMB14CfChlWzvUOD6qnp4eJorjQuXAy9vH+8GbJJkPWBv4LvAkqqaVlUz+tVAaTgdfvjhnHfeeU88P//889l6662ZP38+NCcxXwV8PMmk1djsxsDsqtoVuB84FTgAeCNP7tveBiyuqt2B3YG/SPKcru9HGkvGWri8rarmt4/nAZPbx+cCVNXlwGZJtgD2Af69nX4xcO9QG0xybNs7M/exBxcPZ9ulkeiXVXVl+/jfaQ5eoTkhA7AAuKaq7q+qu4GH2vpaRpIpwEeBIc/0WmvSKpsH7JpkU+BhYA5NyHw5zQmf5bLONBa8+MUv5le/+hWLFi3ihhtuYMstt2T+/PkcccQRAFTV/wI/oAmAq+r3wLfbxwuAH1TVI+3jye30A4GjkswHrgGeRtOxsRTrTOPZxH43YC3r7Q15DBgYFluDlqvlTF9GVZ0NnA2w/qTtV7q8NMYsr3YGau1xlq67xxnieyXJM4GvA0dV1a1DvpC1Jq2SqnokyULgz4CrgBuBVwDPBX68knWtM40J06dPZ9asWdx1110cfvjh3HrrkLuWwR5l6Y6VDXoeP1JVAzXxxL6tqh5PMrBfC/DOqvrOil7EOtN4NtZ6LpfnMIAke9MMZ1hMM6xoRjv9tcCW/WueNGI9O8lL28dHAFes7gbansyLgff29IJK6uZy4F3tzx8CxwHz24PjR9phstKYdfjhh/OVr3yFWbNmMX36dPbZZ58nhsom2YZmhNq1g1ZbCExL8pQkzwL2WM2X/Q7wVwP1leT5STbu9EakMWa8hMt7k1wFnEUzXh7gg8A+Sa6jGebwi341ThrBfgwcneRGYCvgn9dgG+8Angec3N5kZH6Sp6/NRkrj0A+BScCcdgjgQzw5JPZs4EZv6KOxbMqUKdx///1st912TJo0iTe+8Y3svPPOAFOA7wPvqaq7Bq12JXAbzVDXTwDXrebLfh64BbiuvWndZxl7owClTvLkCACtzPqTtq9JR39qrW934WkHrfVtSqsqybyqWuZvwSaZDHyjqnZa121af9L29fCdP1vXLysNq+XVWr8M1z5tpHEfO76M1Drzc6ixZEV1Nl56LiVJkiRJw8iufElDqqqFwDrvtZQkSdLoZM+lJEmSJKkzw6UkSZIkqTPDpSRJkiSpM8OlJEmSJKkzw6UkSZIkqTPDpSRJkiSpM8OlJEmSJKkzw6UkSZIkqTPDpSRJkiSpM8OlJEmSJKkzw6UkSZIkqTPDpSRJkiSpM8OlJEmSJKkzw6UkSZIkqTPDpSRJkiSpM8OlJEmSJKkzw6UkSZIkqTPDpSRJkiSps4n9bsBoMnW7zZl72kH9boY05k3dbvN+N0Ea89ynScPPOtN4Y8+lJEmSJKkzw6UkSZIkqTPDpSRJkiSpM8OlJEmSJKkzw6UkSZIkqTPDpSRJkiSpM8OlJEmSJKkzw6UkSZIkqTPDpSRJkiSpM8OlJEmSJKkzw6UkSZIkqTPDpSRJkiSps4n9bsBosuCOxUw+8eJ18loLTztonbyONBINrjXrQVr71uU+bazwu0iry/2Zxht7LiVJkiRJnRkuJUmSJEmdGS4lSZIkSZ0ZLiVJkiRJnRkuJUmSJEmdGS4lSZIkSZ0ZLiVJkiRJnRkuJUmSJEmdGS4lSZIkSZ0ZLiVJkiRJnRkuJUmSJEmdGS4lSZIkSZ0ZLiVJkiRJnRkuJUmSJEmdGS4lSZIkSZ0ZLiVJkiRJnRkuJUmSJEmdGS4lSZIkSZ0ZLiVJkiRJnRkuJUmSJEmdjdtwmeSAJPOSLGh/vrLfbZLGoiRPS3JZkgeSnNHv9kgjVZItkry93+2QxpMkk5PcNMT02Ul260ebpNFsXIbLJBOBe4CDq2oqcDTwb/1tlTT2tLX2EHAy8K4+N0ca6bYADJeSpFFrYr8bsKaSTAa+BVwB7AXcAbyhnTYf2APYDHhrVV2b5BRgW2AycE9V/WnP5m4GNkiyflU9vI7egjQqrKVauyLJ89Z126VR5jTguUnmA48AS4BfAzsAl9MEzwD/AuwGFDCzqv4pyV8AxwJPBf4bOLKqHlz3b0EalSYk+RxL7+MA3pLkdJbex+0BfArYkKZG/6yqftqPRksj0Wjvudwe+ExVTQHuAw5tp29cVXvR7Ihn9iy/K/CGQcGSdr3rDZbScq2tWpO0fCcCt1bVNODdNCdu/g6YCjwXeBMwDdiuqnZqR958oV33a1W1e1W9CPgx8LZ13npp9FqdfdxPgH2q6sXAB4CPrOvGSiPZqO25bN1WVfPbx/NoekoAzgWoqsuTbJZki3b6RVW1pHcDSaYAHwUOHOoFkhxLczaYCZtts3ZbL40enWttZaw1aRnXVtXPAZKcC+wNfA/4oyT/H3AxcEm77E5JTqUZWrsJ8J2hNmidSUNanX3cpsAXk2xPM3pgvcEbs840no32nsvensbHeDIs16DlBp7/rndikmcCXweOqqpbh3qBqjq7qnarqt0mbLT5WmiyNCp1qrVVYa1Jy1imvqrqXuBFwGzg/wCfb+edA7yj7c38ILDBkBu0zqShrM4+7h+Ay6pqJ+Bghqg160zj2WgPl8tzGECSvYHFVbV48ALt2aeLgfdW1ZXruH3SWLHSWpO0yu6n6RUZsEeS5yR5Ck2tXZFka+ApVfVVmhtl7dIuuylwZ5L1gBnrstHSGDbUPm5zmusyAY7pU7ukEWu0D4tdnnuTXEV7AfZylnkH8Dzg5CQnt9MOrKpfrYsGSmPEqtQaSRa2yzw1ySE0tXbLummiNDpU1a+TXNn+WYQlwByam/xMpbmhz9fbx19oAyfAe9ufJwPXALcDC1g6pEpaM0Pt4z5GMyz2b4Hv961l0gg1asNlVS0Edup5/glo/i4R8NWqeu+g5U8Z9PxU4NThbqc02nWttXba5OFsozRWDNwEK8l+wLuq6rBBi9zAk72Vvev9M/DPw95AaYxZ3j5uBcvPAZ7fM+nk5S0rjUdjdVisJEmSJGkdGrU9l8tTVfv1uw3SeGCtScOnqmbT3LRHkqRRw55LSZIkSVJnhktJkiRJUmeGS0mSJElSZ4ZLSZIkSVJnhktJkiRJUmeGS0mSJElSZ4ZLSZIkSVJnhktJkiRJUmeGS0mSJElSZ4ZLSZIkSVJnhktJkiRJUmeGS0mSJElSZ4ZLSZIkSVJnhktJkiRJUmeGS0mSJElSZ4ZLSZIkSVJnhktJkiRJUmeGS0mSJElSZxP73YDRZOp2mzP3tIP63QxpzLPWpOFnnUnDzzrTeGPPpSRJkiSpM8OlJEmSJKkzw6UkSZIkqTPDpSRJkiSpM8OlJEmSJKkzw6UkSZIkqTPDpSRJkiSpM8OlJEmSJI7ejcYAACAASURBVKkzw6UkSZIkqTPDpSRJkiSpM8OlJEmSJKkzw6UkSZIkqbOJ/W7AaLLgjsVMPvHivrZh4WkH9fX1pXVhcK35uZfWvpGwTxuv/E4bP1ZUZ34ONBbZcylJkiRJ6sxwKUmSJEnqzHApSZIkSerMcClJkiRJ6sxwKUmSJEnqzHApSZIkSerMcClJkiRJ6sxwKUmSJEnqzHApSZIkSerMcClJkiRJ6sxwKUmSJEnqzHApSZIkSerMcClJkiRJ6sxwKUmSJEnqzHApSZIkSerMcClJkiRJ6sxwKUmSJEnqzHApSZIkSerMcClJkiRJ6sxwKUmSJEnqzHAJJHlfv9sgSRpfkkxOclO/2yGNV8urwSSzk+y2CusvTLL18LROGp0Mlw3DpSRJkiR1MC7CZZL3JDm+ffxPSb7fPt4/ySxgwyTzk3yprw2VRqAkb0lybVsjn00yIclrklyX5IYk32uX2yTJF5IsSHJjkkPb6QcmmdMu/x9JNunvO5KGzwknnMCZZ575xPNTTjmFT37yk7z73e8GmNLWx2GD10tyTJIzep5/I8l+7eMHknw0ybwklybZo+1Z+XmS17fLTEjy8SQ/auvvL4f7vUpjxIQkn0tyc5JLkmzYTn9LkquS3JRkD4AkT2uXuT7JZ4H0r9nSyDQuwiVwOfDy9vFuwCZJ1gP2Br4LLKmqaVU1o18NlEaiJC8ADgNeVlXTgMeAtwCfAw6tqhcBb24XPxlYXFVTq2pn4PvtcKH3A6+qql2AucDfruv3Ia0rhx9+OOedd94Tz88//3y23npr5s+fD3Az8Crg40kmrcZmNwZmV9WuwP3AqcABwBuBD7XLvI2m/nYHdgf+Islzur4faRzYHvhMVU0B7gMObadvXFV7AW8HZrbT/h64oqpeDFwEPHtdN1Ya6Sb2uwHryDxg1ySbAg8D19GEzJcDx69oxSTHAscCTNhsm2FupjTi7A/sCvwoCcCGwJ7A5VV1G0BV/aZd9lXA4QMrVtW9SV4HvBC4sl3/qcCcoV7IWtNY8OIXv5hf/epXLFq0iLvvvpstt9yS+fPnc8QRR3DppZdSVf+b5Ac0AfDGVdzs74Fvt48XAA9X1SNJFgCT2+kHAjsnmd4+35zmoPm23g1ZZ9Iybquq+e3jeTxZU+cCVNXlSTZLsgWwD/CmdvrFSe4daoPWmcazcREu253wQuDPgKtoduivAJ4L/Hgl654NnA2w/qTta3hbKo04Ab5YVe99YkIzDO9PlrPs4BoJ8N2qOmJlL2StaayYPn06s2bN4q677uLwww/n1ltvXZXVHmXp0UQb9Dx+pKoGauJxmpOkVNXjSQb24wHeWVXfWdGLWGfSMh7uefwYzUlUWHZ/VsuZvgzrTOPZeBkWC83Q2He1P38IHAfMb3fYj7TDZCUt7XvA9CRPB0iyFXADsO/AkLt2GsAlwDsGVkyyJXA18LIkz2unbZTk+euw/dI6d/jhh/OVr3yFWbNmMX36dPbZZ58nhsom2Yam9+PaQastBKYleUqSZwF7rObLfgf4q4F9WZLnJ9m40xuRxrfDAJLsTTPkfDHNMeSMdvprgS371zxpZBpP4fKHwCRgTlX9L/BQOw2as0s3ekMfaWlVdQvNNZOXJLmR5hrlSTTDfb6W5AZg4AKzU4Et25sf3AC8oqruBo4Bzm3XvxrYcR2/DWmdmjJlCvfffz/bbbcdkyZN4o1vfCM777wzwBTg+8B7ququQatdSTOEdQHwCZrLN1bH54FbgOvaP63wWcbJ6CRpmNyb5CrgLJprmgE+COyT5Dqaoei/6FfjpJEqT4600cqsP2n7mnT0p/rahoWnHdTX19fYk2ReVa3073mtS4Nrzc+9xoKRVmsjYZ82XvmdNnxGU535OdBotaI6G089l5IkSZKkYWK4lCRJkiR1ZriUJEmSJHVmuJQkSZIkdWa4lCRJkiR1ZriUJEmSJHVmuJQkSZIkdWa4lCRJkiR1ZriUJEmSJHVmuJQkSZIkdWa4lCRJkiR1ZriUJEmSJHVmuJQkSZIkdWa4lCRJkiR1ZriUJEmSJHVmuJQkSZIkdWa4lCRJkiR1ZriUJEmSJHVmuJQkSZIkdWa4lCRJkiR1ZriUJEmSJHU2sd8NGE2mbrc5c087qN/NkMY8a00aftaZNPysM4039lxKkiRJkjozXEqSJEmSOjNcSpIkSZI6M1xKkiRJkjozXEqSJEmSOjNcSpIkSZI6M1xKkiRJkjozXEqSJEmSOjNcSpIkSZI6M1xKkiRJkjozXEqSJEmSOpvY7waMJgvuWMzkEy/udzNGnIWnHdTvJmiMWdu15mdUWpb7tLHH77qRZ7jqzP9rjVT2XEqSJEmSOjNcSpIkSZI6M1xKkiRJkjozXEqSJEmSOjNcSpIkSZI6M1xKkiRJkjozXEqSJEmSOjNcSpIkSZI6M1xKkiRJkjozXEqSJEmSOjNcSpIkSZI6M1xKkiRJkjozXEqSJEmSOjNcSpIkSZI6M1xKkiRJkjozXEqSJEmSOjNcSpIkSZI6M1xKkiRJkjozXEqSJEmSOjNcSpIkSZI6M1xKkiRJkjozXEpaa5JctZL5b07y4ySXras2SaNRkslJbhpi+uwku/WjTZIkrcxKw2WS49uDwS91eaEkxyTZtuf555O8sMs2O7RlgyTXJrkhyc1JPtiPdkhjTVXttZJF3ga8vapesS7aI0nqbvLkydxzzz0A7LXX8r/mZ8+ezete97rV2vY555zDokWLOrVvdSV5fZIT1+mLSuPEqvRcvh3446qaMTAhycQ1eK1jgCfCZVX9eVXdsgbbWRseBl5ZVS8CpgGvSfKSPrVFGjOSPND+nJTk8iTzk9yU5OVJPgDsDZyV5OP9bak0KkxI8rn2JOglSTZsp78lyVVtbe0BkGSPdtr17c8d+thujWFXXbXCASqrbU3C5aOPPrrGr5dkYlVdVFWnrfFGJC3XCsNlkrOAPwIuSrI4ydlJLgH+tR2y88Mk17X/9upZ7z1JFrQ9g6clmQ7sBnypPdjcsHdoT5Ij2uVvSvLRnu08kOTD7XauTvIHK2jrm9v1b0hyeTvtmCQXJvl2kp8m+XuAajzQrrpe+6/W5BcoaUh/CnynqqYBLwLmV9WHgLnAjKp6d19bJ40O2wOfqaopwH3Aoe30jdtRAm8HZrbTfgLsU1UvBj4AfGRdN1ZjwyGHHMKuu+7KlClTOPvss5eZv8kmm6xw/QceeIDp06ez4447MmPGDKqaw6t58+ax7777suuuu/LqV7+aO++8k1mzZjF37lxmzJjBtGnTWLJkyZDLAey33368733vY9999+XTn/40t99+O/vvvz8777wz+++/P7/4xS+W26Yk5yT5f+0lGR9tjw/P6Jl3entS5uftMStJnpLkzPbkzjeSfLNn3mlJbklyY5JPrMnvWRqrVtgDWVXHJXkN8ArgHcDBwN5VtSTJRsABVfVQku2Bc4HdkrwWOATYs6oeTLJVVf0myTuAd1XVXIAktD+3BT4K7ArcC1yS5JCqugDYGLi6qk5K8jHgL4BTl9PcDwCvrqo7kmzRM30PYCfgQeBHSS6uqrlJJgDzgOfR7LyvGWqjSY4FjgWYsNk2K/p1SXrSj4CZSdYDLqiq+StbwVqTlnFbT+3MAya3j88FqKrLk2zW7vM2Bb7Y7o+L5qTpMqwzrczMmTPZaqutWLJkCbvvvjuHHnroylfqcf3113PzzTez7bbb8rKXvYwrr7ySPffck3e+851ceOGFbLPNNpx33nmcdNJJzJw5kzPOOINPfOIT7LbbbjzyyCPLXQ7gvvvu4wc/+AEABx98MEcddRRHH300M2fO5Pjjj+eCCy5YUdOeD7yqqh5LcsygeZNoRtbsCFwEzALeRFNzU4GnAz+m2a9tBbwR2LGqatAxJ2CdaXxb3eGtF1XVkvbxesAZSaYBj9EULcCrgC9U1YMAVfWblWxzd2B2Vd0N0F7buQ9wAfB74BvtcvOAA1awnSuBc5KcD3ytZ/p3q+rX7ba/RvPlMbeqHgOmtV8KX0+yU1Utc/OEqjobOBtg/Unb27sprYL2oHcf4CDg35J8vKr+dSXrWGvS0h7uefwYMDAsdnB9FPAPwGVV9cYkk4HZQ23QOtPKnH766Xz9618H4Je//CU/+9nPVmv9PfbYg2c+85kATJs2jYULF7LFFltw0003ccABzWHcY489xqRJk5ZZ96c//ekKlzvssMOeeDxnzhy+9rXmcO/II4/kPe95z8qa9h/tsd9QLqiqx4FbekbJ7d2u8zhwV568Ed1vgYeAzye5mCePU59gnWk8W91w+buex38D/C/NkLen0BQaQFi9IaZZwbxHamA8RbNjXW57217WPWkOZue3oZch2lKD1rsvyWzgNcAy4VLS6kvyh8AdVfW5JBsDuwArDJeSVtlhwGVJ9gYWV9XiJJsDd7Tzj+lbyzSqzZ49m0svvZQ5c+aw0UYbsd9++/HQQw+tfMUe66+//hOPJ0yYwKOPPkpVMWXKFObMmbPCdVe23MYbb7zcdQdGxK3A71Ywr/dETgb9XEpVPdpe67w/cDjNyL5XruzFpfGiy58i2Ry4sz2jcyQwoZ1+CfDWdtgs7fABgPtphu0Mdg2wb5Kt26GqRwA/WN3GJHluVV1TVR8A7gGe1c46IMlW7Y0QDgGuTLLNwDCGdvqraK5XkbR27Edzkud6muvEPt3f5khjyr1p/uzPWTR3YAb4GPCPSa7kyf2xtFoWL17MlltuyUYbbcRPfvITrr766rWy3R122IG77777idD4yCOPcPPNNwOw6aabcv/99690ucH22msvvvKVrwDwpS99ib333nuttLXHFcCh7bWXf0CzXyPJJsDmVfVN4K9pbgwpqbUmd30dcCbw1SRvBi6jPSNUVd9uew3nJvk98E3gfcA5NHeJXAK8dGAjVXVnkve22wjwzaq6cA3a8/H2WpMA3wNuoCn4K4B/o7m28svt9ZY701ybMoEmYJ9fVcsMa5C0eqpqk/bnF4EvDjF/v3XdJmk0qqqFNPcLGHi+wpuGVNUcnrw8BeDk4WmZxrLXvOY1nHXWWey8887ssMMOvOQla+dG+k996lOZNWsWxx9/PIsXL+bRRx/lr//6r5kyZQrHHHMMxx13HBtuuCFz5sxZ7nKDnX766bz1rW/l4x//ONtssw1f+MIX1kpbe3yVpnfyJuC/aDpDFtN0lFyYZAOaY86/WdsvLI1meXLU6djTXrC9W1W9Y21sb/1J29ekoz+1NjY1piw87aB+N0EdJJlXVSPqj7Kv7VrzM6qRYKTVmvu0scfvurVbZ0k2qaoHkjwNuBZ4WVXdtTrbGK468/9a/bSiOuvScylJkiSNVd9oL6N6KvAPqxsspfFo1IXLJCcBbx40+T+q6sODl62qc2iG40qSJGktWLBgAUceeeRS09Zff32uuWbIv+q2znz4wx8GeGGS3j9/NeQx4qrwUg5p9Y26cNl+QazRl4QkSZK6mTp1KvPnr/TPF69zJ510Eu9///tvGUnDz6XxpsvdYiVJkiRJAgyXkiRJkqS1wHApSZIkSerMcClJkiRJ6sxwKUmSJEnqzHApSZIkSerMcClJkiRJ6sxwKUmSJEnqzHApSZIkSerMcClJkiRJ6sxwKUmSJEnqzHApSZIkSerMcClJkiRJ6sxwKUmSJEnqzHApSZIkSerMcClJkiRJ6mxivxswmkzdbnPmnnZQv5shjXnWmjT8rDNp+FlnGm/suZQkSZIkdWa4lCRJkiR1ZriUJEmSJHVmuJQkSZIkdWa4lCRJkiR1ZriUJEmSJHVmuJQkSZIkdWa4lCRJkiR1ZriUJEmSJHVmuJQkSZIkdWa4lCRJkiR1ZriUJEmSJHU2sd8NGE0W3LGYySde3O9mjDkLTzuo303QCGOtjRzW59hlnWko1vzaNZx15v+VRiJ7LiVJkiRJnRkuJUmSJEmdGS4lSZIkSZ0ZLiVJkiRJnRkuJUmSJEmdGS4lSZIkSZ0ZLiVJkiRJnRkuJUmSJEmdGS4lSZIkSZ0ZLiVJkiRJnRkuJUmSJEmdGS4lSZIkSZ0ZLiVJkiRJnRkuJUmSJEmdGS4lSZIkSZ0ZLiVJkiRJnRkuJUmSJEmdGS4lSZIkSZ0ZLiVJkiRJnRkuJUmSJEmdjdtwmeRpSS5L8kCSM/rdHkmSJI0eST6f5IX9boc0kozLcJlkIvAQcDLwrj43RwLg9NNP5wUveAEzZszotJ1zzjmHRYsWPfH8z//8z7nlllu6Nm+NJNkgybVJbkhyc5IP9qUhGpMmT57MPffcA8Bee+213OVmz57N6173utXa9uA6WheSvD7Jiev0RSWtsar686rqzw5WGqEm9rsBayrJZOBbwBXAXsAdwBvaafOBPYDNgLdW1bVJTgG2BSYD91TVnwJXJHneum67NJQzzzyTb33rWzznOc95Ytqjjz7KxImrV6bnnHMOO+20E9tuuy0An//859dqO1fTw8Arq+qBJOvR1Ny3qurqfjZKY89VV121Vrc3uI5WxZrU64AkE6vqIuCiNdqApGGVZGPgfOCZwATgH4C/At5VVXP72TZpJBntPZfbA5+pqinAfcCh7fSNq2ov4O3AzJ7ldwXe0AZLacQ47rjj+PnPf87rX/96Nt98c4499lgOPPBAjjrqKBYuXMjLX/5ydtllF3bZZZelDqI/9rGPMXXqVF70ohdx4oknMmvWLObOncuMGTOYNm0aS5YsYb/99mPu3Ga/d+655zJ16lR22mknTjjhhCe20w4P/3Dbw3h1kj9YXluTvDnJTe2yl7fTjklyYZJvJ/lpkr8HqMYD7arrtf9qbf/+NPYdcsgh7LrrrkyZMoWzzz57mfmbbLLJCtd/4IEHmD59OjvuuCMzZsygqvkYzps3j3333Zddd92VV7/61dx5551D1tFQywHst99+vO9972Pffffl05/+NLfffjv7778/O++8M/vvvz+/+MUvltumJOck+X9JLgM+2tbRGT3zTk9yVZKfJ5neTn9KkjPbkQDfSPLNnnmnJbklyY1JPrEmv2dJy/UaYFFVvaiqdgK+3e8GSSPRqO25bN1WVfPbx/NoeiUBzgWoqsuTbJZki3b6RVW1ZHVeIMmxwLEAEzbbpnuLpSGcddZZfPvb3+ayyy7jjDPO4D//8z+54oor2HDDDXnwwQf57ne/ywYbbMDPfvYzjjjiCObOncu3vvUtLrjgAq655ho22mgjfvOb37DVVltxxhln8IlPfILddtttqddYtGgRJ5xwAvPmzWPLLbfkwAMP5IILLhiYvTFwdVWdlORjwF8Apy6nuR8AXl1Vd/TUFjSjBXYCHgR+lOTiqpqbZAJNfT6P5mTQNUNt1FrTisycOZOtttqKJUuWsPvuu3PooYeufKUe119/PTfffDPbbrstL3vZy7jyyivZc889eec738mFF17INttsw3nnncdJJ53EzJkzl6qjRx55ZLnL/f/t3X2UZHV95/H3h4EZGB5VCGcE46CLeHgSdURQ1gwGDUIEjWYxIQjBjTtBwqqbdWWJiXlQMXqiEl2QGMSnQOTJcHRXDcYBQQRBBmaGBxGZXfAhSIQRJSAM3/3j3mZqerpnuud2d1V1v1/n1Olbt3516/u7Vd+q+63fr24DPPjgg1x55ZUAvPrVr+aNb3wjJ554Iueddx6nnXZab56N5TnAEVW1LslJo25bBBwGPJdmRPNi4LdoPusOAH4FuA04L8lTgdcCz62qGpWbTzLPpC22EvhgkvcDX6yqbyQZs6F5prls2IvLR3uW1wHbtcujR0ZGrv9isg9QVecC5wIsWLS3Iy6aEccccwzbbde8nB977DFOPfVUVqxYwbx58/jud78LwBVXXMHv//7vs3DhQgCe+tSnbnKb3/72t1m6dCm77dZ80B1//PFcddVVIzf/Evhiu3wj8IpNbOoa4Pwknwcu7Vn/z1X1bwBJLqU5KL6hqtYBB7UHu5cl2b+qVo3eqLmmTTnrrLO47LLLALjnnnu48847J3X/gw8+mD333BOAgw46iDVr1rDLLruwatUqXvGK5uW+bt06Fi1atNF977jjjk22O+64455cvvbaa7n00iYtTjjhBN7xjndsLrSL2hwZyxeq6gng1p7ZBIe193kC+HE76gnwM5pzCXwiyZdYn88bMM+kLVNV303yQuAo4H1JvrqJtuaZ5qxhLy7Hcxzw9SSHAWurau143y5Jg2j77bd/cvlDH/oQu+++OzfffDNPPPEE2267LQBVxWRe1yPTAMfxWK1vsI5NvDdU1bIkLwaOBlYkOWjkptFNR93vwSTLaaYWbVRcSuNZvnw5V1xxBddeey0LFy5k6dKlPPLII5PaxoIFC55cnjdvHo8//jhVxX777ce11167yfturl1vvo42gRzd1JeevV+gZtTfDVTV40kOBn4deANwKvDyzT24pIlJ8nTgp1X12SQ/B07qc0jSQBr231yO54Ek3wTOAd40XqMka4C/AU5Kcq+nk9YgWrt2LYsWLWKrrbbiM5/5DOvWNYMcr3zlKznvvPN4+OGHAfjpT38KwI477shDDz200XZe/OIXc+WVV3L//fezbt06LrjgAn7t135t0vEkeXZVXVdVfwrcDzyjvekVSZ6aZDvgNcA1SXYbmZ7Xrj8CuH3SD6o5be3atTzlKU9h4cKF3H777XzrW1NzPqh99tmHn/zkJ08WjY899hirV68GNsyjTbUb7SUveQkXXnghAJ/73Oc47LDDpiTWHlcDr2t/e7k7sBQgyQ7AzlX1v4G3AgeNvwlJW+AA4PokK4AzGP+nI9KcNrQjl1W1hub3XSPXPwjQjoxcUlWnj2r/7jG2sXg6Y5SmwimnnMLrXvc6LrroIg4//PAnR0mOPPJIVqxYwZIlS5g/fz5HHXUU733veznppJNYtmwZ22233QYjLYsWLeJ973sfhx9+OFXFUUcdxbHHHrslIX0gyd40IyhfA26mOZC9GvgMzW8r/6H9veWBwKfa311uBXy+qsacrieN58gjj+Scc87hwAMPZJ999uGQQw6Zku3Onz+fiy++mNNOO421a9fy+OOP89a3vpX99ttvozwar91oZ511FieffDIf+MAH2G233fjkJz85JbH2uIRmdHIV8F3gOmAtsCPwT0m2pcnNt031A0tzWVV9BfjKqNVL+xCKNNCymalyQ6ctLqfltNALFu1di0788FRvds5bc+bR/Q5hTktyY1Ut2XzLTW7jJGBJVZ06FTGZa4PD/Jw6U5Fr7XZ2aP+9z9OA64GXVtWPJ7sd80xjGfacn6o8myrTmWfD/lxpeG0qz4Z25HI8VbW03zFIkjSNvthON58P/OWWFJaSJE2HWVdcSpoaSc4AfnvU6ouq6j2j21bV+cD5MxCWNK6VK1dywgknbLBuwYIFXHfdmP/9Zsa85z3v4aKLLgLYt/29FoyTSxPhl6iSpEFlcSlpTO2B7xYd/Er9cMABB7BixYrNN5xhZ5xxBmeccQZJbh2k6XqSJE212Xq2WEmSJEnSDLK4lCRJkiR1ZnEpSZIkSerM4lKSJEmS1JnFpSRJkiSpM4tLSZIkSVJnFpeSJEmSpM4sLiVJkiRJnVlcSpIkSZI6s7iUJEmSJHVmcSlJkiRJ6sziUpIkSZLUmcWlJEmSJKkzi0tJkiRJUmcWl5IkSZKkziwuJUmSJEmdbd3vAIbJAXvszA1nHt3vMKRZz1yTpp95Jk0/80xzjSOXkiRJkqTOLC4lSZIkSZ1ZXEqSJEmSOrO4lCRJkiR1ZnEpSZIkSerM4lKSJEmS1JnFpSRJkiSpM4tLSZIkSVJnFpeSJEmSpM4sLiVJkiRJnVlcSpIkSZI6s7iUJEmSJHW2db8DGCYrf7CWxe/8Ur/DkMa15syj+x3ClDDXNCyGOefMM02VYc6D6TZb88znXONx5FKSJEmS1JnFpSRJkiSpM4tLSZIkSVJnFpeSJEmSpM4sLiVJkiRJnVlcSpIkSZI6s7iUJEmSJHVmcSlJkiRJ6sziUpIkSZLUmcWlJEmSJKkzi0tJkiRJUmcWl5IkSZKkziwuJUmSJEmdWVxKkiRJkjqzuJQkSZIkdWZxKUmSJEnqzOJSkiRJktSZxaUkSZIkqTOLS0mSJElSZxaXkiRJkqTO5mxxmeTdSf6433FIwyDJLklO6XccktZL8vQkF7fLBydZ0V5uTvLafscnzTZJnpbk60l+nuSj/Y5HGkRztriUNCm7ABaX0gCpqh9W1evbq6uAJVV1EHAk8PEkW/cvOml2afPpEeBdgIMT0jiGtrhM8oUkNyZZneTNSeYlOT/JqiQrk7ytbbc8yYeTfLO97eCezezb3v79JKf1qSvSMDgTeHY7KvLtJFcluSzJrUnOSbLVJnLwD9r73JzkkiQL+9wXaWAkWZzk9iSfSnJLkouTLEyyJsl7k1yb5IYkL0jylSR3JVnWc99VAFX1cFU93m52W6D61SdpULU5c1uSv2uPH7+aZLvxjhXbWW7nJvkq8Omq+kVVXU1TZEoawzB/q3lyVf00yXbAt4EbgT2qan9opvH1tN2+ql6S5GXAecD+7frnAocDOwJ3JDm7qh6buS5IQ+OdwP5VdVCSpcCXgX2B/9su/xZwN2Pn4KVV9Xftur8C3gT87cyGLw20fYA3VdU1Sc5j/SyBe6rq0CQfAs4HXkpTOK4Gzhm9kSQvpvmMeyZwQk+xKWm9vYHfqao/SPJ54HXt+vGOFV8IHFZV/96HWKWhM7Qjl8BpSW4GvgU8A5gPPCvJ3yY5EvhZT9sLAKrqKmCnnoPeL1XVo1V1P3AfsPvoB2lHRW9IcsO6h9dOZ3+kYXJ9VX2/qtbR5NdhwPcZOwf3T/KNJCuB44H9xtqguaY57J6quqZd/ixNPgFc3v5dCVxXVQ9V1U+AR0Z9gQpAVV1XVfsBLwJOT7Lt6DbmmcTdVbWiXb4RWNwuj3esePlkC0vz/7ZlfAAAFItJREFUTHPZUBaX7cjJEcChVfU84CZgAfA8YDnwFuATPXcZPT1o5PqjPevWMcZIblWdW1VLqmrJvIU7T0n80iywUU5V1QOMnYPnA6dW1QHAn9OMvGy8QXNNc9fmPqOeYMPPqyfYxMyjqroN+AXrR156bzPPNNeNd+w3Xh7+YrIPYJ5pLhvK4hLYGXigqh5O8lzgEGBXYKuquoTmx9Yv6Gl/HECSw4C1VeXXSNLkPEQzfXzEwUn2SrIVTX5dnWS8HNwR+FGSbWhGLiVt6FeTHNou/w5w9WQ30Obj1u3yM2mm2q6Zsgil2c9jRWkKDOtvLr8MLEtyC3AHzdTYPYDl7cEuwOk97R9I8k1gJ+DkGY1UmgWq6t+SXNOePOTfgWtpTvJzAHAVcFm7/MkxcvBdwHU0v89cyYZFqiS4DTgxyceBO4GzgT+a5DYOA96Z5DGakc1T2p98SJqYCR0rJlnTtpmf5DXAK6vq1pkJURp8Q1lcVtWjwKvGuOkj49zlkqrqLTapqnePur7R9CFJ61XV78KT09L/uKqOG9XkZjacMTByv7NpDpYlje2Jqlo2at3ikYWqOp9mevnI9ZHb7qed+lpVnwE+M40xSkOvqtbQM128qj4IzX8WYALHiu26xdMZozTshnVarCRJkiRpgAzlyOVkVNXSfscgzSZVtZzmpD2SOho9kiJp5nmsKE0dRy4lSZIkSZ1ZXEqSJEmSOrO4lCRJkiR1ZnEpSZIkSerM4lKSJEmS1JnFpSRJkiSpM4tLSZIkSVJnFpeSJEmSpM4sLiVJkiRJnVlcSpIkSZI6s7iUJEmSJHVmcSlJkiRJ6sziUpIkSZLUmcWlJEmSJKkzi0tJkiRJUmcWl5IkSZKkziwuJUmSJEmdWVxKkiRJkjqzuJQkSZIkdbZ1vwMYJgfssTM3nHl0v8OQZj1zTZp+5pk0/cwzzTWOXEqSJEmSOrO4lCRJkiR1ZnEpSZIkSerM4lKSJEmS1JnFpSRJkiSpM4tLSZIkSVJnFpeSJEmSpM4sLiVJkiRJnVlcSpIkSZI6s7iUJEmSJHVmcSlJkiRJ6mzrfgcwTFb+YC2L3/mlfochTcqaM4/udwiTZq5p2Jhn0vQzz6SZ0SXXHLmUJEmSJHVmcSlJkiRJ6sziUpIkSZLUmcWlJEmSJKkzi0tJkiRJUmcWl5IkSZKkziwuJUmSJEmdWVxKkiRJkjqzuJQkSZIkdWZxKUmSJEnqzOJSkiRJktSZxaUkSZIkqTOLS0mSJElSZxaXkiRJkqTOLC4lSZIkSZ1ZXEqSJEmSOrO4lCRJkiR1ZnEpSZIkSerM4lKSJEmS1JnFpSRJkiSpM4tLSZIkSVJnQ1VcJlmcZFW/45DmsvHyMMnyJEv6EZM0FyV5epKL2+VXJLkxycr278v7HZ802yR5d5I/7ncc0iDbut8BSJKkyauqHwKvb6/eD7y6qn6YZH/gK8AefQtOkjQn9X3kMsn7k5zSc/3dSf5bkg8kWdV+C3vcGPc7KclHe65/McnSdvnn7XZvTHJFkoPbUZXvJzmmbTOvfYxvJ7klyX+Zge5Ks8W8JH+XZHWSrybZrl3/e0m+2ebuwQBt/n0zyU3t3336GLc0UNqZALcn+VT7WXRxkoVJ1iR5b5Jrk9yQ5AVJvpLkriTLeu67CqCqbmqLTYDVwLZJFvSrX9IgSvKF9thwdZI3t8eC5/ccb76tbbc8yYdHf5619u05pjytT12RBlbfi0vgQqC3ePxPNN/AHgQ8DzgC+ECSRZPY5vbA8qp6IfAQ8FfAK4DXAn/RtnkTsLaqXgS8CPiDJHt16Yg0h+wNfKyq9gMeBF7Xrt++ql4CnAKc1667HXhZVT0f+FPgvTMdrDTg9gHOraoDgZ/R5A/APVV1KPAN4HyaUcpDWP85Np7XATdV1aPTE640tE5ujw2XAKfRHGvuUVX7V9UBwCd72o71eQbwXOA3gIOBP0uyzcyELg2Hvk+LraqbkvxKkqcDuwEP0CT7BVW1DvjXJFfSFIC3THCzvwS+3C6vBB6tqseSrAQWt+tfCRyYZGRK0c40B8x3924oyZuBNwPM22m3LeihNCvdXVUr2uUbWZ9XFwBU1VVJdkqyC7Aj8KkkewMFjPlBbK5pDrunqq5plz9Lc9ALcHn7dyWwQ1U9BDyU5JE2tzaSZD/g/TSfcWPdbp5pLjstyWvb5WcA84FnJflb4EvAV3vajvV5BvCl9oubR5PcB+wO3Nv7IOaZ5rJBGLkEuJjmG9njaEYyM4H7PM6G8W/bs/xYVVW7/ATwKEBVPcH6gjrAH1XVQe1lr6rqfVOhvc+5VbWkqpbMW7jzpDolzWK9IyLrWJ9XNapdAX8JfL2q9gdezYa5ur6huaa5a6y8gfV59gQb5lzvZ9mTkuwJXAa8saruGvOBzDPNUe1Pp44ADq2q5wE3AQtoZsktB94CfKLnLpvLS9jw8299Q/NMc9igFJcXAm+gKTAvBq4Cjmvnwu8GvAy4ftR91gAHJdkqyTNopidMxleAPxyZzpDkOUm279AHSe0U9ySH0Uw7X0szK+AH7e0n9SkuaZD9apJD2+XfAa6e7AbaUZUvAaf3jIJKWm9n4IGqejjJc2mmmO8KbFVVlwDvAl7Q036szzNJm9H3abEAVbU6yY7AD6rqR0kuAw4Fbqb5pugdVfXjJIt77nYNzRTWlcAq4DuTfNhP0Ezl+06SAD8BXtOlH5J4IMk3gZ2Ak9t1f00zLfbtwL/0LTJpcN0GnJjk48CdwNnAH01yG6cC/wF4V5J3teteWVX3TV2Y0lD7MrAsyS3AHcC3aM6ovDzJyGDL6T3tx/o8k7QZA1FcArQ/pB5ZLuC/t5feNmuA/XvaHD/OtnboWX73WLe1U2T/Z3uRNEG9edhe/+Bm2l8LPKdn1bvGayvNUU9U1bJR6xaPLFTV+TQn9Bm5PnLb/az/TPwrmpPXSRpD+zvJV41x00fGucslVdVbbI51TLk/kjYwKNNiJUmSJElDbGBGLiVJmmtGzwSQ1H9VtbTfMUjDypFLSZIkSVJnFpeSJEmSpM4sLiVJkiRJnVlcSpIkSZI6s7iUJEmSJHVmcSlJkiRJ6sziUpIkSZLUmcWlJEmSJKkzi0tJkiRJUmcWl5IkSZKkziwuJUmSJEmdWVxKkiRJkjqzuJQkSZIkdWZxKUmSJEnqzOJSkiRJktSZxaUkSZIkqTOLS0mSJElSZxaXkiRJkqTOtu53AMPkgD125oYzj+53GNKsZ65J0888k6afeaa5xpFLSZIkSVJnFpeSJEmSpM4sLiVJkiRJnVlcSpIkSZI6s7iUJEmSJHVmcSlJkiRJ6sziUpIkSZLUmcWlJEmSJKkzi0tJkiRJUmcWl5IkSZKkziwuJUmSJEmdWVxKkiRJkjqzuJQkSZIkdWZxKUmSJEnqzOJSkiRJktSZxaUkSZIkqTOLS0mSJElSZxaXkiRJkqTOLC4lSZIkSZ1ZXEqSJEmSOrO4lCRJkiR1ZnEpSZIkSerM4lKSJEmS1JnFpSRJkiSps1RVv2MYGkkeAu7odxwzaFfg/n4HMYPman+fWVW79TuYXubanDDX+rwrsP0g5doQ5dkwvVaGJdZhiRMmH+tAfaYNUZ6NmM2vjX4bpng3F+u4ebb19MQza91RVUv6HcRMSXKD/Z29Bry/5tosN9f63PZ3cb/jGGUo8myYXivDEuuwxAnDFes4hiLPRgzT/h6mWGG44u0Sq9NiJUmSJEmdWVxKkiRJkjqzuJycc/sdwAyzv7PbIPd3kGObDnOtvzD3+jyI/R3EmMYyLHHC8MQ6LHHCcMU6lmGLf5jiHaZYYbji3eJYPaGPJEmSJKkzRy4lSZIkSZ1ZXI4hyZFJ7kjyvSTvHOP2BUn+sb39uiSLZz7KqTOB/r4syXeSPJ7k9f2IcSpNoL9vT3JrkluSfC3JM/sR51SZQH+XJVmZZEWSq5PsO8OPP24+JTm9XX9Hkt+Y6Db7bZr6vKbnebphZnoyMVva3yRPS/L1JD9P8tFR93lh29/vJTkrSWamNxMzTX1e3m5zRXv5lX7F3y9JntHun9uSrE7yX9v1T03yz0nubP8+pd+xAiSZl+SmJF9sr+/VPt93ts///H7HCJBklyQXJ7m93beHDuI+TfK29nlfleSCJNsO6j6diEHNs7EkOS/JfUlW9TuWzRnvfWIQta/h65Pc3Mb65/2OaXNGv69NWlV56bkA84C7gGcB84GbgX1HtTkFOKddfgPwj/2Oe5r7uxg4EPg08Pp+xzwD/T0cWNgu/+EceH536lk+BvjyDD/+mPkE7Nu2XwDs1W5n3kS2OQT7fFJ9bm9bA+za7/5NcX+3Bw4DlgEfHXWf64FDgQD/B3hVv/s6A31eDiwZhPj7uG8XAS9ol3cEvtvmxV8D72zXvxN4f79jbWN5O/APwBfb658H3tAunwP8Yb9jbGP5FPCf2+X5wC6Dtk+BPYC7ge169uVJg7pPJ9Cfgc2zceJ9GfACYFW/Y5lArGO+T/Q7rnFiDbBDu7wNcB1wSL/j2kzMG7yvTfbiyOXGDga+V1Xfr6pfAhcCx45qcyzNGzXAxcCvD9q36pOw2f5W1ZqqugV4oh8BTrGJ9PfrVfVwe/VbwJ4zHONUmkh/f9ZzdXtgKn+I3SWfjgUurKpHq+pu4Hvt9iayzX6ajj4Psi3ub1X9oqquBh7pbZxkEc2XHtdW80n3aeA109qLyZnyPs+wgc2hqvpRVX2nXX4IuI2m6Ojdn59iAF4PSfYEjgY+0V4P8HKa5xsGJ86daAqHvweoql9W1YMM4D6l+f/r2yXZGlgI/IgB3KcTNLB5Npaqugr4ab/jmIhNvE8MnGr8vL26TXsZ2BPejH5f2xIWlxvbA7in5/q9bPyCfbJNVT0OrAWeNiPRTb2J9Hc2mWx/30QzajKsJtTfJG9JchfNN9mnzfDjj5dP49130F+z09FnaD6MvprkxiRvnoa4t9R0vGfu0W5nU9vsp+n8nPhkOyX2XdP4peWg5xAA7VTi59N80797Vf0ImgNLYNqmDE/Ch4F3sP6L16cBD7bPNwzOfn0W8BOa19ZNST6RZHsGbJ9W1Q+ADwL/j6aoXAvcyGDu04kYijwbdqPeJwZSO810BXAf8M9VNbCxsvH72qRZXG5srA/z0d8wTKTNsJhNfZmICfc3ye8BS4APTGtE02tC/a2qj1XVs4H/AfzJDD/+eG0mu35QTEefAV5aVS8AXgW8JcnLtjzEKTUd75mz+TnelOOr6gDgP7aXE7YgtokY9P1Lkh2AS4C3jppdMRCS/CZwX1Xd2Lt6jKaDsF+3ppnueHZVPR/4Bc002IHS/ubzWJqfBDydZibNq8ZoOgj7dCIG9fUwawz6+8SIqlpXVQfRzIQ7OMn+/Y5pLOO8r02axeXG7gWe0XN9T+CH47Vpp27szJBMJRjDRPo7m0yov0mOAM4AjqmqR2cotukw2ef3QqZ2ylGXfBrvvoP+mp2OPlNVI3/vAy5jcKbLTsd75r1sOB19Nj3H42pHbkamef0D0/ccD3QOJdmG5oDxc1V1abv6X9vp0iPTpu/rV3ytlwLHJFlD8775cppv/Hdpn28YnP16L3Bvz2jJxTTF5qDt0yOAu6vqJ1X1GHAp8BIGc59OxEDn2bAb531ioLXT0ZcDR/Y5lPFs9L6W5LOT3YjF5ca+Dezdnp1sPs2JGC4f1eZy4MR2+fXAv7S/CxpGE+nvbLLZ/iZ5PvBxmsKy3x+2XU2kv3v3XD0auHMmH5/x8+ly4A1pzrq5F7A3zUleBv01O+V9TrJ9kh0B2ulsrwQG5Yx+U/6e2U7ReyjJIe3U0DcC/zT1oW+xKe9zkq2T7NoubwP8JtP3HA9sDrXP998Dt1XV3/Tc1Ls/T6TPr4eqOr2q9qyqxTT771+q6njg6zTPNwxAnABV9WPgniT7tKt+HbiVAdunNNNhD0mysH0djMQ5cPt0ggY2z4bdJt4nBk6S3ZLs0i5vR/Mlyu39jWps47yv/d6WbMjLxmdJOormzFN3AWe06/6CptgA2Ba4iOZkG9cDz+p3zNPc3xfRfAP3C+DfgNX9jnma+3sF8K/AivZyeb9jnub+fgRY3fb168B+M/z44+YTzejxXcAd9JwtdKxtDtJlqvtM85upm9vL6kHrc8f+rqEZ0ft5+z6zb7t+CU1xdRfwUSD97ud09plmCuCNwC3tc/wR2jMFz1T8g3ChOZNutfth5D34KJrfM36N5suvrwFP7XesPTEvZf3ZYp/VPt/fa5//Bf2Or43rIOCGdr9+AXjKIO5T4M9pDrxXAZ+hOXP2QO7TCfZnIPNsnFgvoPmt62Pt+9Kb+h3TJmId832i33GNE+uBwE1trKuAP+13TBOM+8n3tcle0m5AkiRJkqQt5rRYSZIkSVJnFpeSJEmSpM4sLiVJkiRJnVlcSpIkSZI6s7iUJEmSpFkkyXlJ7kuy2X9rlWRZkpVJViS5Osm+PbednuR7Se5I8hub3ZZni5UkSZKk2SPJy2j+5dWnq2r/zbTdqap+1i4fA5xSVUe2ReYFwMHA02n+Xd9zqmrdeNty5FKSJEmSZpGquormfyo/Kcmzk3w5yY1JvpHkuW3bn/U0257m/4gCHAtcWFWPVtXdNP9v9uBNPe7WU9YDSZIkSdKgOhdYVlV3Jnkx8L+AlwMkeQvwdmD+yDpgD+BbPfe/t103LotLSZIkSZrFkuwAvAS4KMnI6gUjC1X1MeBjSX4X+BPgRCCjt8P6Uc0xWVxKkiRJ0uy2FfBgVR20mXYXAme3y/cCz+i5bU/gh5t7EEmSJEnSLNX+rvLuJL8NkMbz2uW9e5oeDdzZLl8OvCHJgiR7AXsD12/qcRy5lCRJkqRZJMkFwFJg1yT3An8GHA+cneRPgG1oRilvBk5NcgTwGPAAzZRYqmp1ks8DtwKPA2/Z1JliwX9FIkmSJEmaAk6LlSRJkiR1ZnEpSZIkSerM4lKSJEmS1JnFpSRJkiSpM4tLSZIkSVJnFpeSJEmSpM4sLiVJkiRJnVlcSpIkSZI6+/+GFmeGMqSTrAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1080x720 with 4 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, imp_plot = plt.subplots(nrows=1, ncols=4, figsize=(15, 10))\n",
    "imp_plot[0].barh(importances.nlargest(8).index.values, importances.nlargest(8))\n",
    "imp_plot[0].set_title('Dummy Model')\n",
    "imp_plot[1].barh(scores_mutual.nlargest(8).index.values, scores_mutual.nlargest(8))\n",
    "imp_plot[1].set_title('Mutual Information')\n",
    "imp_plot[2].barh(scores_f.nlargest(8).index.values, scores_f.nlargest(8))\n",
    "imp_plot[2].set_title('F Regression')\n",
    "imp_plot[3].barh(importances_lasso.nlargest(8).index.values, importances_lasso.nlargest(8))\n",
    "imp_plot[3].set_title('LASSO')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GradientBoosting\n",
    "### Model-chosen features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-16T12:29:32.771430Z",
     "start_time": "2020-01-16T12:29:32.765407Z"
    }
   },
   "outputs": [],
   "source": [
    "gbr_param_grid = {\n",
    "    'n_estimators': [500, 1000, 2000],\n",
    "    'max_depth': [2, 4, 6],\n",
    "    'min_samples_leaf': [3, 5, 9, 17],\n",
    "    'learning_rate': [0.1, 0.05, 0.02],\n",
    "    'max_features': [1.0, 0.3, 0.1],\n",
    "    'loss': ['ls', 'lad', 'huber']\n",
    "} #hyperparameters for each gbr grid search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-17T13:32:17.742207Z",
     "start_time": "2020-01-17T13:32:09.059129Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Score:\t 0.9999999992944544\n",
      "Testing Score:\t -0.0008312552035061315\n",
      "GradientBoostingRegressor run at: 2020-01-17 13:32:17.722153 . Saving to models/RDKit_VPAS/GradientBoostingRegressor_model_17_01_2020_13_32_17.joblib\n"
     ]
    }
   ],
   "source": [
    "X_train, X_test = X_train_main[top_15_features_model], X_test_main[top_15_features_model]\n",
    "\n",
    "\n",
    "best_hp = grid_search_return_hp_dict(GradientBoostingRegressor(),\n",
    "                                     #X_train, y_train, gbr_param_grid)\n",
    "\n",
    "gbr_model = GradientBoostingRegressor()\n",
    "gbr_model.fit(X_train, y_train)\n",
    "\n",
    "exp_vs_calc_gbr_model = produce_exp_vs_pred_df('top_15_features_model', gbr_model,\n",
    "                                         \"GradBoost, model-chosen features\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### F-regression features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-16T14:33:39.603510Z",
     "start_time": "2020-01-16T13:31:54.739984Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GradientBoostingRegressor grid search.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'learning_rate': 0.05, 'loss': 'lad', 'max_depth': 6, 'max_features': 0.1, 'min_samples_leaf': 3, 'n_estimators': 500}\n",
      "Training Score:\t 0.9317395172085381\n",
      "Testing Score:\t 0.8411981283092483\n",
      "GradientBoostingRegressor run at: 2020-01-16 14:33:39.559393 . Saving to models/DRAGON_noVPAS/GradientBoostingRegressor_kbest_16_01_2020_14_33_39.joblib\n"
     ]
    }
   ],
   "source": [
    "X_train, X_test = X_train_main[top_15_features_kbest], X_test_main[top_15_features_kbest]\n",
    "\n",
    "best_hp = grid_search_return_hp_dict(GradientBoostingRegressor(),\n",
    "                                     X_train, y_train, gbr_param_grid)\n",
    "\n",
    "gbr_freg = GradientBoostingRegressor(**best_hp)\n",
    "gbr_freg.fit(X_train, y_train)\n",
    "\n",
    "exp_vs_calc_gbr_freg = produce_exp_vs_pred_df('top_15_features_kbest', gbr_freg,\n",
    "                                                    \"GradBoost, f_reg features\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Mutual info features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-16T15:38:51.251936Z",
     "start_time": "2020-01-16T14:33:39.607527Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GradientBoostingRegressor grid search.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'learning_rate': 0.05, 'loss': 'lad', 'max_depth': 6, 'max_features': 0.3, 'min_samples_leaf': 3, 'n_estimators': 2000}\n",
      "Training Score:\t 0.9379608336062498\n",
      "Testing Score:\t 0.8544833146624162\n",
      "GradientBoostingRegressor run at: 2020-01-16 15:38:51.112718 . Saving to models/DRAGON_noVPAS/GradientBoostingRegressor_mutual_info_16_01_2020_15_38_51.joblib\n"
     ]
    }
   ],
   "source": [
    "X_train, X_test = X_train_main[top_15_features_mutual_info], X_test_main[top_15_features_mutual_info]\n",
    "\n",
    "\n",
    "best_hp = grid_search_return_hp_dict(GradientBoostingRegressor(),\n",
    "                                     X_train, y_train, gbr_param_grid)\n",
    "\n",
    "gbr_mu = GradientBoostingRegressor(**best_hp)\n",
    "gbr_mu.fit(X_train, y_train)\n",
    "\n",
    "exp_vs_calc_gbr_mu = produce_exp_vs_pred_df('top_15_features_mutual_info', gbr_mu,\n",
    "                                            \"GradBoost, Mutual Info\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LASSO features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-16T16:37:07.166038Z",
     "start_time": "2020-01-16T15:38:51.256449Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GradientBoostingRegressor grid search.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'learning_rate': 0.02, 'loss': 'ls', 'max_depth': 6, 'max_features': 0.3, 'min_samples_leaf': 3, 'n_estimators': 2000}\n",
      "Training Score:\t 0.996589735091017\n",
      "Testing Score:\t 0.8161460167193553\n",
      "GradientBoostingRegressor run at: 2020-01-16 16:37:06.990069 . Saving to models/DRAGON_noVPAS/GradientBoostingRegressor_lasso_16_01_2020_16_37_06.joblib\n"
     ]
    }
   ],
   "source": [
    "X_train, X_test = X_train_main[top_15_features_lasso], X_test_main[top_15_features_lasso]\n",
    "\n",
    "best_hp = grid_search_return_hp_dict(GradientBoostingRegressor(), \n",
    "                                     X_train, y_train, gbr_param_grid)\n",
    "\n",
    "gbr_lasso = GradientBoostingRegressor(**best_hp)\n",
    "gbr_lasso.fit(X_train, y_train)\n",
    "exp_vs_calc_gbr_lasso = produce_exp_vs_pred_df('top_15_features_lasso', gbr_lasso,\n",
    "                                               \"LASSO-features, GBR model\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Decision Tree\n",
    "### Model-Chosen Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-16T16:37:07.180076Z",
     "start_time": "2020-01-16T16:37:07.172555Z"
    }
   },
   "outputs": [],
   "source": [
    "dtr_param_grid = {\n",
    "    'criterion': ['mse', 'friedman_mse', 'mae'], #function measuring quality of a split\n",
    "    'max_depth': [5, 10, 20] , # max depth of tree\n",
    "    'max_features': [5, 10, 15], # N features to be considered when looking for split\n",
    "    'max_leaf_nodes': [None, 10, 15, 20, 30], #grows tree with N nodes in best-first fashion\n",
    "    'min_impurity_decrease': [0.0, 0.1],\n",
    "    'min_samples_leaf': [1, 2, 3], \n",
    "    'min_samples_split': [1.0, 2, 3],\n",
    "    'min_weight_fraction_leaf': [0.0, 0.1, 0.5] , \n",
    "    'splitter': ['best', 'random']\n",
    "} #hyperparameters for dtr gridsearch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-16T16:40:54.361628Z",
     "start_time": "2020-01-16T16:37:07.182580Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DecisionTreeRegressor grid search.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n",
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:814: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'criterion': 'mse', 'max_depth': 20, 'max_features': 15, 'max_leaf_nodes': None, 'min_impurity_decrease': 0.0, 'min_samples_leaf': 3, 'min_samples_split': 3, 'min_weight_fraction_leaf': 0.0, 'splitter': 'random'}\n",
      "Training Score:\t 0.9141647290701933\n",
      "Testing Score:\t 0.7956771880980333\n",
      "DecisionTreeRegressor run at: 2020-01-16 16:40:54.355856 . Saving to models/DRAGON_noVPAS/DecisionTreeRegressor_model_16_01_2020_16_40_54.joblib\n"
     ]
    }
   ],
   "source": [
    "X_train, X_test = X_train_main[top_15_features_model], X_test_main[top_15_features_model]\n",
    "\n",
    "best_hp = grid_search_return_hp_dict(DecisionTreeRegressor(), X_train, y_train, dtr_param_grid)\n",
    "\n",
    "dtr_model = DecisionTreeRegressor(**best_hp)\n",
    "\n",
    "dtr_model.fit(X_train, y_train)\n",
    "\n",
    "exp_vs_calc_dtr_model = produce_exp_vs_pred_df('top_15_features_model', dtr_model,\n",
    "                                         \"Decision Tree, model-chosen features\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### F-regression features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-16T16:44:48.342240Z",
     "start_time": "2020-01-16T16:40:54.364602Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DecisionTreeRegressor grid search.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n",
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:814: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'criterion': 'mae', 'max_depth': 10, 'max_features': 10, 'max_leaf_nodes': None, 'min_impurity_decrease': 0.0, 'min_samples_leaf': 3, 'min_samples_split': 2, 'min_weight_fraction_leaf': 0.0, 'splitter': 'random'}\n",
      "Training Score:\t 0.824735879420048\n",
      "Testing Score:\t 0.7707477970128914\n",
      "DecisionTreeRegressor run at: 2020-01-16 16:44:48.335723 . Saving to models/DRAGON_noVPAS/DecisionTreeRegressor_kbest_16_01_2020_16_44_48.joblib\n"
     ]
    }
   ],
   "source": [
    "X_train, X_test = X_train_main[top_15_features_kbest], X_test_main[top_15_features_kbest]\n",
    "\n",
    "\n",
    "best_hp = grid_search_return_hp_dict(DecisionTreeRegressor(), X_train, y_train, dtr_param_grid)\n",
    "\n",
    "dtr_freg = DecisionTreeRegressor(**best_hp)\n",
    "\n",
    "dtr_freg.fit(X_train, y_train)\n",
    "\n",
    "exp_vs_calc_dtr_freg = produce_exp_vs_pred_df('top_15_features_kbest', dtr_freg,\n",
    "                                         \"Decision Tree, freg features\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Mutual info features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-16T16:49:12.282181Z",
     "start_time": "2020-01-16T16:44:48.346251Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DecisionTreeRegressor grid search.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n",
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:814: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'criterion': 'friedman_mse', 'max_depth': 10, 'max_features': 10, 'max_leaf_nodes': None, 'min_impurity_decrease': 0.1, 'min_samples_leaf': 3, 'min_samples_split': 2, 'min_weight_fraction_leaf': 0.0, 'splitter': 'random'}\n",
      "Training Score:\t 0.8595461536744464\n",
      "Testing Score:\t 0.7715041249308991\n",
      "DecisionTreeRegressor run at: 2020-01-16 16:49:12.276174 . Saving to models/DRAGON_noVPAS/DecisionTreeRegressor_mutual_info_16_01_2020_16_49_12.joblib\n"
     ]
    }
   ],
   "source": [
    "X_train, X_test = X_train_main[top_15_features_mutual_info], X_test_main[top_15_features_mutual_info]\n",
    "\n",
    "best_hp = grid_search_return_hp_dict(DecisionTreeRegressor(), X_train, y_train, dtr_param_grid)\n",
    "\n",
    "dtr_mu = DecisionTreeRegressor(**best_hp)\n",
    "\n",
    "dtr_mu.fit(X_train, y_train)\n",
    "\n",
    "exp_vs_calc_dtr_mu = produce_exp_vs_pred_df('top_15_features_mutual_info', dtr_mu,\n",
    "                                         \"Decision Tree, mutual info features\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LASSO features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-16T16:53:05.178607Z",
     "start_time": "2020-01-16T16:49:12.286191Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DecisionTreeRegressor grid search.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n",
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:814: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'criterion': 'mae', 'max_depth': 10, 'max_features': 10, 'max_leaf_nodes': None, 'min_impurity_decrease': 0.0, 'min_samples_leaf': 2, 'min_samples_split': 3, 'min_weight_fraction_leaf': 0.0, 'splitter': 'best'}\n",
      "Training Score:\t 0.8840999565633477\n",
      "Testing Score:\t 0.7448102791928264\n",
      "DecisionTreeRegressor run at: 2020-01-16 16:53:05.172089 . Saving to models/DRAGON_noVPAS/DecisionTreeRegressor_lasso_16_01_2020_16_53_05.joblib\n"
     ]
    }
   ],
   "source": [
    "X_train, X_test = X_train_main[top_15_features_lasso], X_test_main[top_15_features_lasso]\n",
    "\n",
    "best_hp = grid_search_return_hp_dict(DecisionTreeRegressor(), X_train, y_train, dtr_param_grid)\n",
    "\n",
    "dtr_lasso = DecisionTreeRegressor(**best_hp)\n",
    "\n",
    "dtr_lasso.fit(X_train, y_train)\n",
    "\n",
    "exp_vs_calc_dtr_lasso = produce_exp_vs_pred_df('top_15_features_lasso', dtr_lasso,\n",
    "                                         \"Decision Tree, lasso features\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## AdaBoost Regressor\n",
    "### Model Chosen Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-16T16:53:05.190138Z",
     "start_time": "2020-01-16T16:53:05.182116Z"
    }
   },
   "outputs": [],
   "source": [
    "base_models = [ExtraTreesRegressor(n_estimators= 5,\n",
    "                                   criterion= 'mse',\n",
    "                                   max_features = 'log2'),\n",
    "               RandomForestRegressor(n_estimators= 5,\n",
    "                                     criterion= 'mse',\n",
    "                                     max_features = 'sqrt',\n",
    "                                     min_samples_split = 3),\n",
    "              GradientBoostingRegressor(),\n",
    "              DecisionTreeRegressor(),\n",
    "              Lars(),\n",
    "              ElasticNet()]\n",
    "\n",
    "ada_param_grid = {\n",
    "    'base_estimator' : base_models, \n",
    "    'learning_rate' : [0.3, 0.5, 0.8, 1], \n",
    "    'loss' : ['linear', 'square', 'exponential'], \n",
    "    'n_estimators' : [50, 100]\n",
    "} #hyperparameters for adaboost gridsearch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-16T17:01:08.672880Z",
     "start_time": "2020-01-16T16:53:05.194650Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AdaBoostRegressor grid search.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n",
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:814: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'base_estimator': ExtraTreesRegressor(bootstrap=False, criterion='mse', max_depth=None,\n",
      "                    max_features='log2', max_leaf_nodes=None,\n",
      "                    min_impurity_decrease=0.0, min_impurity_split=None,\n",
      "                    min_samples_leaf=1, min_samples_split=2,\n",
      "                    min_weight_fraction_leaf=0.0, n_estimators=5, n_jobs=None,\n",
      "                    oob_score=False, random_state=None, verbose=0,\n",
      "                    warm_start=False), 'learning_rate': 0.8, 'loss': 'square', 'n_estimators': 50}\n",
      "Training Score:\t 0.9966830903147815\n",
      "Testing Score:\t 0.8520725299365427\n",
      "AdaBoostRegressor run at: 2020-01-16 17:01:08.526492 . Saving to models/DRAGON_noVPAS/AdaBoostRegressor_model_16_01_2020_17_01_08.joblib\n"
     ]
    }
   ],
   "source": [
    "X_train, X_test = X_train_main[top_15_features_model], X_test_main[top_15_features_model]\n",
    "\n",
    "best_hp = grid_search_return_hp_dict(AdaBoostRegressor(), X_train, y_train, ada_param_grid)\n",
    "\n",
    "ada_model = AdaBoostRegressor(**best_hp)\n",
    "\n",
    "ada_model.fit(X_train, y_train)\n",
    "\n",
    "exp_vs_calc_ada_model = produce_exp_vs_pred_df('top_15_features_model', ada_model,\n",
    "                                               \"AdaBoost, model features\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### F-regression features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-16T17:09:18.037626Z",
     "start_time": "2020-01-16T17:01:08.683411Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AdaBoostRegressor grid search.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n",
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:814: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'base_estimator': ExtraTreesRegressor(bootstrap=False, criterion='mse', max_depth=None,\n",
      "                    max_features='log2', max_leaf_nodes=None,\n",
      "                    min_impurity_decrease=0.0, min_impurity_split=None,\n",
      "                    min_samples_leaf=1, min_samples_split=2,\n",
      "                    min_weight_fraction_leaf=0.0, n_estimators=5, n_jobs=None,\n",
      "                    oob_score=False, random_state=None, verbose=0,\n",
      "                    warm_start=False), 'learning_rate': 0.8, 'loss': 'exponential', 'n_estimators': 50}\n",
      "Training Score:\t 0.9979723730365371\n",
      "Testing Score:\t 0.8433963130222173\n",
      "AdaBoostRegressor run at: 2020-01-16 17:09:17.903303 . Saving to models/DRAGON_noVPAS/AdaBoostRegressor_kbest_16_01_2020_17_09_17.joblib\n"
     ]
    }
   ],
   "source": [
    "X_train, X_test = X_train_main[top_15_features_kbest], X_test_main[top_15_features_kbest]\n",
    "\n",
    "best_hp = grid_search_return_hp_dict(AdaBoostRegressor(), X_train, y_train, ada_param_grid)\n",
    "\n",
    "ada_freg = AdaBoostRegressor(**best_hp)\n",
    "\n",
    "ada_freg.fit(X_train, y_train)\n",
    "\n",
    "exp_vs_calc_ada_freg = produce_exp_vs_pred_df('top_15_features_kbest', ada_freg,\n",
    "                                               \"AdaBoost, freg features\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Mutual info features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-16T17:18:49.503938Z",
     "start_time": "2020-01-16T17:09:18.044141Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AdaBoostRegressor grid search.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n",
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:814: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'base_estimator': ExtraTreesRegressor(bootstrap=False, criterion='mse', max_depth=None,\n",
      "                    max_features='log2', max_leaf_nodes=None,\n",
      "                    min_impurity_decrease=0.0, min_impurity_split=None,\n",
      "                    min_samples_leaf=1, min_samples_split=2,\n",
      "                    min_weight_fraction_leaf=0.0, n_estimators=5, n_jobs=None,\n",
      "                    oob_score=False, random_state=None, verbose=0,\n",
      "                    warm_start=False), 'learning_rate': 0.8, 'loss': 'exponential', 'n_estimators': 100}\n",
      "Training Score:\t 0.9997045578005971\n",
      "Testing Score:\t 0.8365182791277487\n",
      "AdaBoostRegressor run at: 2020-01-16 17:18:49.229088 . Saving to models/DRAGON_noVPAS/AdaBoostRegressor_mutual_info_16_01_2020_17_18_49.joblib\n"
     ]
    }
   ],
   "source": [
    "X_train, X_test = X_train_main[top_15_features_mutual_info], X_test_main[top_15_features_mutual_info]\n",
    "\n",
    "best_hp = grid_search_return_hp_dict(AdaBoostRegressor(), X_train, y_train, ada_param_grid)\n",
    "\n",
    "ada_mu = AdaBoostRegressor(**best_hp)\n",
    "\n",
    "ada_mu.fit(X_train, y_train)\n",
    "\n",
    "exp_vs_calc_ada_mu = produce_exp_vs_pred_df('top_15_features_mutual_info', ada_mu,\n",
    "                                               \"AdaBoost, mutual info features\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LASSO features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-16T17:26:34.176980Z",
     "start_time": "2020-01-16T17:18:49.506946Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AdaBoostRegressor grid search.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n",
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:814: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'base_estimator': RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=None,\n",
      "                      max_features='sqrt', max_leaf_nodes=None,\n",
      "                      min_impurity_decrease=0.0, min_impurity_split=None,\n",
      "                      min_samples_leaf=1, min_samples_split=3,\n",
      "                      min_weight_fraction_leaf=0.0, n_estimators=5, n_jobs=None,\n",
      "                      oob_score=False, random_state=None, verbose=0,\n",
      "                      warm_start=False), 'learning_rate': 0.5, 'loss': 'exponential', 'n_estimators': 50}\n",
      "Training Score:\t 0.9926354574810684\n",
      "Testing Score:\t 0.8080655222178783\n",
      "AdaBoostRegressor run at: 2020-01-16 17:26:34.046134 . Saving to models/DRAGON_noVPAS/AdaBoostRegressor_lasso_16_01_2020_17_26_34.joblib\n"
     ]
    }
   ],
   "source": [
    "X_train, X_test = X_train_main[top_15_features_lasso], X_test_main[top_15_features_lasso]\n",
    "\n",
    "best_hp = grid_search_return_hp_dict(AdaBoostRegressor(), X_train, y_train, ada_param_grid)\n",
    "\n",
    "ada_lasso = AdaBoostRegressor(**best_hp)\n",
    "\n",
    "ada_lasso.fit(X_train, y_train)\n",
    "\n",
    "exp_vs_calc_ada_lasso = produce_exp_vs_pred_df('top_15_features_lasso', ada_lasso,\n",
    "                                               \"AdaBoost, model features\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LASSO\n",
    "### Model Chosen Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-16T17:26:34.188511Z",
     "start_time": "2020-01-16T17:26:34.180996Z"
    }
   },
   "outputs": [],
   "source": [
    "lasso_param_grid = {\n",
    "    'alpha':[0.2, 0.4, 0.6, 0.8],\n",
    "    'max_iter':[1000, 5000, 10000, 50000],\n",
    "    'selection':['cyclic', 'random']\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-16T17:26:40.982758Z",
     "start_time": "2020-01-16T17:26:34.192020Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lasso grid search.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'alpha': 0.2, 'max_iter': 1000, 'selection': 'cyclic'}\n",
      "Training Score:\t 0.7025515174084993\n",
      "Testing Score:\t 0.7357933019197427\n",
      "Lasso run at: 2020-01-16 17:26:40.950674 . Saving to models/DRAGON_noVPAS/Lasso_model_16_01_2020_17_26_40.joblib\n"
     ]
    }
   ],
   "source": [
    "X_train, X_test = X_train_main[top_15_features_model], X_test_main[top_15_features_model]\n",
    "best_hp = grid_search_return_hp_dict(Lasso(), X_train, y_train, lasso_param_grid)\n",
    "lasso_model = Lasso(**best_hp)\n",
    "lasso_model.fit(X_train, y_train)\n",
    "\n",
    "exp_vs_calc_lasso_model = produce_exp_vs_pred_df('top_15_features_model', lasso_model,\n",
    "                                                 \"LASSO, model features\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### F-regression features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-16T17:26:48.532748Z",
     "start_time": "2020-01-16T17:26:40.986267Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lasso grid search.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n",
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\coordinate_descent.py:475: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Duality gap: 62.95777664966954, tolerance: 7.861287863090288\n",
      "  positive)\n",
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\coordinate_descent.py:475: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Duality gap: 58.1372885694509, tolerance: 7.861287863090288\n",
      "  positive)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'alpha': 0.2, 'max_iter': 5000, 'selection': 'random'}\n",
      "Training Score:\t 0.7247376293705728\n",
      "Testing Score:\t 0.7171477393839785\n",
      "Lasso run at: 2020-01-16 17:26:48.526732 . Saving to models/DRAGON_noVPAS/Lasso_kbest_16_01_2020_17_26_48.joblib\n"
     ]
    }
   ],
   "source": [
    "X_train, X_test = X_train_main[top_15_features_kbest], X_test_main[top_15_features_kbest]\n",
    "best_hp = grid_search_return_hp_dict(Lasso(), X_train, y_train, lasso_param_grid)\n",
    "lasso_freg = Lasso(**best_hp)\n",
    "lasso_freg.fit(X_train, y_train)\n",
    "\n",
    "exp_vs_calc_lasso_freg = produce_exp_vs_pred_df('top_15_features_kbest', lasso_freg,\n",
    "                                                 \"LASSO, f-reg features\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Mutual info features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-16T17:26:56.287005Z",
     "start_time": "2020-01-16T17:26:48.536258Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lasso grid search.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'alpha': 0.2, 'max_iter': 5000, 'selection': 'cyclic'}\n",
      "Training Score:\t 0.6858987473879607\n",
      "Testing Score:\t 0.6714929045146443\n",
      "Lasso run at: 2020-01-16 17:26:56.280489 . Saving to models/DRAGON_noVPAS/Lasso_mutual_info_16_01_2020_17_26_56.joblib\n"
     ]
    }
   ],
   "source": [
    "X_train, X_test = X_train_main[top_15_features_mutual_info], X_test_main[top_15_features_mutual_info]\n",
    "best_hp = grid_search_return_hp_dict(Lasso(), X_train, y_train, lasso_param_grid)\n",
    "lasso_mu = Lasso(**best_hp)\n",
    "lasso_mu.fit(X_train, y_train)\n",
    "\n",
    "exp_vs_calc_lasso_mu = produce_exp_vs_pred_df('top_15_features_mutual_info', lasso_mu,\n",
    "                                                 \"LASSO, mutual info features\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LASSO Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-16T17:27:02.310949Z",
     "start_time": "2020-01-16T17:26:56.291518Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lasso grid search.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'alpha': 0.2, 'max_iter': 1000, 'selection': 'cyclic'}\n",
      "Training Score:\t 0.6833898806906213\n",
      "Testing Score:\t 0.6737594517258707\n",
      "Lasso run at: 2020-01-16 17:27:02.303930 . Saving to models/DRAGON_noVPAS/Lasso_lasso_16_01_2020_17_27_02.joblib\n"
     ]
    }
   ],
   "source": [
    "X_train, X_test = X_train_main[top_15_features_lasso], X_test_main[top_15_features_lasso]\n",
    "best_hp = grid_search_return_hp_dict(Lasso(), X_train, y_train, lasso_param_grid)\n",
    "lasso_lasso = Lasso(**best_hp)\n",
    "lasso_lasso.fit(X_train, y_train)\n",
    "\n",
    "exp_vs_calc_lasso_lasso = produce_exp_vs_pred_df('top_15_features_lasso', lasso_lasso,\n",
    "                                                 \"LASSO, lasso features\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Random Forest Regressor\n",
    "### Model Chosen Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-16T17:27:02.321476Z",
     "start_time": "2020-01-16T17:27:02.314459Z"
    }
   },
   "outputs": [],
   "source": [
    "random_forest_param_grid = {\n",
    "    'max_depth': [10, 20, 30, 40, 50, 60, 70, 80, 90, None],\n",
    "    'max_features': ['auto', 'sqrt'],\n",
    "    'min_samples_leaf': [1, 2, 4],\n",
    "    'min_samples_split': [2, 5, 10],\n",
    "    'n_estimators': [200, 400, 600, 800, 1000, 1200, 1400, 1600]\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-16T21:45:19.253824Z",
     "start_time": "2020-01-16T19:42:39.311353Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForestRegressor grid search.\n",
      "{'max_depth': 90, 'max_features': 'sqrt', 'min_samples_leaf': 1, 'min_samples_split': 2, 'n_estimators': 600}\n",
      "Training Score:\t 0.9774090544159082\n",
      "Testing Score:\t 0.8582042794564964\n",
      "RandomForestRegressor run at: 2020-01-16 21:45:18.928355 . Saving to models/DRAGON_noVPAS/RandomForestRegressor_model_16_01_2020_21_45_18.joblib\n"
     ]
    }
   ],
   "source": [
    "X_train, X_test = X_train_main[top_15_features_model], X_test_main[top_15_features_model]\n",
    "best_hp = grid_search_return_hp_dict(RandomForestRegressor(),\n",
    "                                     X_train, y_train, random_forest_param_grid)\n",
    "random_forest_model = RandomForestRegressor(**best_hp)\n",
    "random_forest_model.fit(X_train, y_train)\n",
    "exp_vs_calc_random_forest_model = produce_exp_vs_pred_df('top_15_features_model', random_forest_model,\n",
    "                                                         \"RandomForest, model features\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### F-regression features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-17T01:11:15.484470Z",
     "start_time": "2020-01-16T23:02:41.821348Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForestRegressor grid search.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'max_depth': None, 'max_features': 'sqrt', 'min_samples_leaf': 1, 'min_samples_split': 2, 'n_estimators': 200}\n",
      "Training Score:\t 0.9721752491187455\n",
      "Testing Score:\t 0.8401657504277648\n",
      "RandomForestRegressor run at: 2020-01-17 01:11:15.379690 . Saving to models/DRAGON_noVPAS/RandomForestRegressor_kbest_17_01_2020_01_11_15.joblib\n"
     ]
    }
   ],
   "source": [
    "X_train, X_test = X_train_main[top_15_features_kbest], X_test_main[top_15_features_kbest]\n",
    "best_hp = grid_search_return_hp_dict(RandomForestRegressor(),\n",
    "                                     X_train, y_train, random_forest_param_grid)\n",
    "random_forest_freg = RandomForestRegressor(**best_hp)\n",
    "random_forest_freg.fit(X_train, y_train)\n",
    "exp_vs_calc_random_forest_freg = produce_exp_vs_pred_df('top_15_features_kbest', random_forest_freg,\n",
    "                                                         \"RandomForest, freg features\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Mutual info features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-17T03:47:08.728085Z",
     "start_time": "2020-01-17T01:11:26.589418Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForestRegressor grid search.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'max_depth': None, 'max_features': 'sqrt', 'min_samples_leaf': 1, 'min_samples_split': 2, 'n_estimators': 600}\n",
      "Training Score:\t 0.9729434173898541\n",
      "Testing Score:\t 0.8415611230823128\n",
      "RandomForestRegressor run at: 2020-01-17 03:47:08.433069 . Saving to models/DRAGON_noVPAS/RandomForestRegressor_mutual_info_17_01_2020_03_47_08.joblib\n"
     ]
    }
   ],
   "source": [
    "X_train, X_test = X_train_main[top_15_features_mutual_info], X_test_main[top_15_features_mutual_info]\n",
    "best_hp = grid_search_return_hp_dict(RandomForestRegressor(),\n",
    "                                     X_train, y_train, random_forest_param_grid)\n",
    "random_forest_mu = RandomForestRegressor(**best_hp)\n",
    "random_forest_mu.fit(X_train, y_train)\n",
    "exp_vs_calc_random_forest_mu = produce_exp_vs_pred_df('top_15_features_mutual_info', random_forest_mu,\n",
    "                                                         \"RandomForest, mutual info features\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LASSO features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-17T05:55:00.396379Z",
     "start_time": "2020-01-17T03:47:19.592342Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RandomForestRegressor grid search.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'max_depth': 40, 'max_features': 'sqrt', 'min_samples_leaf': 1, 'min_samples_split': 2, 'n_estimators': 200}\n",
      "Training Score:\t 0.9714625413023873\n",
      "Testing Score:\t 0.810773797396149\n",
      "RandomForestRegressor run at: 2020-01-17 05:55:00.292476 . Saving to models/DRAGON_noVPAS/RandomForestRegressor_lasso_17_01_2020_05_55_00.joblib\n"
     ]
    }
   ],
   "source": [
    "X_train, X_test = X_train_main[top_15_features_lasso], X_test_main[top_15_features_lasso]\n",
    "best_hp = grid_search_return_hp_dict(RandomForestRegressor(),\n",
    "                                     X_train, y_train, random_forest_param_grid)\n",
    "random_forest_lasso = RandomForestRegressor(**best_hp)\n",
    "random_forest_lasso.fit(X_train, y_train)\n",
    "exp_vs_calc_random_forest_lasso = produce_exp_vs_pred_df('top_15_features_lasso', random_forest_lasso,\n",
    "                                                         \"RandomForest, lasso features\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LARS - Least angle regression\n",
    "### Model chosen features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-17T05:55:14.660635Z",
     "start_time": "2020-01-17T05:55:11.579996Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Score:\t -5.118873363102049\n",
      "Testing Score:\t -4.66823156501682\n",
      "Lars run at: 2020-01-17 05:55:14.653620 . Saving to models/DRAGON_noVPAS/Lars_model_17_01_2020_05_55_14.joblib\n"
     ]
    }
   ],
   "source": [
    "X_train, X_test = X_train_main[top_15_features_model], X_test_main[top_15_features_model]\n",
    "\n",
    "lars_model = Lars(fit_intercept=False, n_nonzero_coefs=np.inf)\n",
    "lars_model.fit(X_train, y_train)\n",
    "exp_vs_calc_lars_model = produce_exp_vs_pred_df('top_15_features_model', lars_model, \"LARS model features\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### F-regression features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-17T05:55:28.069282Z",
     "start_time": "2020-01-17T05:55:25.653510Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Score:\t 0.412098008114794\n",
      "Testing Score:\t 0.36890756929970014\n",
      "Lars run at: 2020-01-17 05:55:28.063264 . Saving to models/DRAGON_noVPAS/Lars_kbest_17_01_2020_05_55_28.joblib\n"
     ]
    }
   ],
   "source": [
    "X_train, X_test = X_train_main[top_15_features_kbest], X_test_main[top_15_features_kbest]\n",
    "\n",
    "lars_freg = Lars(fit_intercept=False, n_nonzero_coefs=np.inf)\n",
    "lars_freg.fit(X_train, y_train)\n",
    "exp_vs_calc_lars_freg = produce_exp_vs_pred_df('top_15_features_kbest', lars_freg, \"LARS freg features\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Mutual info features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-17T05:55:41.948880Z",
     "start_time": "2020-01-17T05:55:39.134989Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Score:\t -4.03397309481404\n",
      "Testing Score:\t -3.6461882169232736\n",
      "Lars run at: 2020-01-17 05:55:41.943866 . Saving to models/DRAGON_noVPAS/Lars_mutual_info_17_01_2020_05_55_41.joblib\n"
     ]
    }
   ],
   "source": [
    "X_train, X_test = X_train_main[top_15_features_mutual_info], X_test_main[top_15_features_mutual_info]\n",
    "\n",
    "lars_mu = Lars(fit_intercept=False, n_nonzero_coefs=np.inf)\n",
    "lars_mu.fit(X_train, y_train)\n",
    "exp_vs_calc_lars_mu = produce_exp_vs_pred_df('top_15_features_mutual_info', lars_mu, \"LARS mutual info features\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LASSO features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-17T05:55:56.222599Z",
     "start_time": "2020-01-17T05:55:53.060731Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Score:\t 0.6395259500652293\n",
      "Testing Score:\t 0.6194334350344687\n",
      "Lars run at: 2020-01-17 05:55:56.217585 . Saving to models/DRAGON_noVPAS/Lars_lasso_17_01_2020_05_55_56.joblib\n"
     ]
    }
   ],
   "source": [
    "X_train, X_test = X_train_main[top_15_features_lasso], X_test_main[top_15_features_lasso]\n",
    "\n",
    "lars_lasso = Lars(fit_intercept=False, n_nonzero_coefs=np.inf)\n",
    "lars_lasso.fit(X_train, y_train)\n",
    "exp_vs_calc_lars_lasso = produce_exp_vs_pred_df('top_15_features_lasso', lars_lasso, \"LARS lasso features\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Elastic Net (linear regression l1 l2 norm regularization)\n",
    "### Model chosen features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-17T05:56:07.757325Z",
     "start_time": "2020-01-17T05:56:07.750305Z"
    }
   },
   "outputs": [],
   "source": [
    "net_param_grid = {\n",
    "    'alpha':[0.1, 0.3, 0.5, 0.8, 1.0],\n",
    "    'l1_ratio':[0.0, 0.2, 0.4, 0.6, 0.8, 1.0],\n",
    "    'fit_intercept':[True, False],\n",
    "    'max_iter': [1000, 3000, 5000],\n",
    "    'tol':[0.0001, 0.001, 0.01],\n",
    "    'selection':['cyclic', 'random']\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-17T05:56:58.966814Z",
     "start_time": "2020-01-17T05:56:19.112124Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ElasticNet grid search.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'alpha': 0.1, 'fit_intercept': False, 'l1_ratio': 0.8, 'max_iter': 3000, 'selection': 'cyclic', 'tol': 0.0001}\n",
      "Training Score:\t 0.7134538033619403\n",
      "Testing Score:\t 0.7453632055451128\n",
      "ElasticNet run at: 2020-01-17 05:56:58.961798 . Saving to models/DRAGON_noVPAS/ElasticNet_model_17_01_2020_05_56_58.joblib\n"
     ]
    }
   ],
   "source": [
    "X_train, X_test = X_train_main[top_15_features_model], X_test_main[top_15_features_model]\n",
    "best_hp = grid_search_return_hp_dict(ElasticNet(), X_train, y_train, net_param_grid)\n",
    "\n",
    "net_model = ElasticNet(**best_hp)\n",
    "net_model.fit(X_train, y_train)\n",
    "exp_vs_calc_net_model = produce_exp_vs_pred_df('top_15_features_model', net_model, \"ElasticNet model features\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### F regression features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-17T05:58:12.294870Z",
     "start_time": "2020-01-17T05:57:10.374599Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ElasticNet grid search.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n",
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\coordinate_descent.py:475: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Duality gap: 10978.14596703574, tolerance: 78.61287863090288\n",
      "  positive)\n",
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\coordinate_descent.py:475: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Duality gap: 10979.363160643945, tolerance: 78.61287863090288\n",
      "  positive)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'alpha': 0.1, 'fit_intercept': True, 'l1_ratio': 0.0, 'max_iter': 3000, 'selection': 'random', 'tol': 0.001}\n",
      "Training Score:\t 0.7332180807485775\n",
      "Testing Score:\t 0.7301068054649074\n",
      "ElasticNet run at: 2020-01-17 05:58:12.288854 . Saving to models/DRAGON_noVPAS/ElasticNet_kbest_17_01_2020_05_58_12.joblib\n"
     ]
    }
   ],
   "source": [
    "X_train, X_test = X_train_main[top_15_features_kbest], X_test_main[top_15_features_kbest]\n",
    "best_hp = grid_search_return_hp_dict(ElasticNet(), X_train, y_train, net_param_grid)\n",
    "\n",
    "net_freg = ElasticNet(**best_hp)\n",
    "net_freg.fit(X_train, y_train)\n",
    "exp_vs_calc_net_freg = produce_exp_vs_pred_df('top_15_features_kbest', net_freg, \"ElasticNet freg features\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Mutual info features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-17T05:59:22.498246Z",
     "start_time": "2020-01-17T05:58:23.823817Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ElasticNet grid search.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n",
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\coordinate_descent.py:475: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Duality gap: 113.7454182191832, tolerance: 7.941628996607655\n",
      "  positive)\n",
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\coordinate_descent.py:475: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Duality gap: 113.7454182191832, tolerance: 7.941628996607655\n",
      "  positive)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'alpha': 0.1, 'fit_intercept': False, 'l1_ratio': 1.0, 'max_iter': 5000, 'selection': 'cyclic', 'tol': 0.0001}\n",
      "Training Score:\t 0.6970302526384997\n",
      "Testing Score:\t 0.6787535164371025\n",
      "ElasticNet run at: 2020-01-17 05:59:22.492231 . Saving to models/DRAGON_noVPAS/ElasticNet_mutual_info_17_01_2020_05_59_22.joblib\n"
     ]
    }
   ],
   "source": [
    "X_train, X_test = X_train_main[top_15_features_mutual_info], X_test_main[top_15_features_mutual_info]\n",
    "best_hp = grid_search_return_hp_dict(ElasticNet(), X_train, y_train, net_param_grid)\n",
    "\n",
    "net_mu = ElasticNet(**best_hp)\n",
    "net_mu.fit(X_train, y_train)\n",
    "exp_vs_calc_net_mu = produce_exp_vs_pred_df('top_15_features_mutual_info', net_mu, \"ElasticNet mutual info features\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LASSO features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-17T06:00:05.016950Z",
     "start_time": "2020-01-17T05:59:34.404713Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ElasticNet grid search.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:1978: FutureWarning: The default value of cv will change from 3 to 5 in version 0.22. Specify it explicitly to silence this warning.\n",
      "  warnings.warn(CV_WARNING, FutureWarning)\n",
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\coordinate_descent.py:475: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Duality gap: 12487.608190337094, tolerance: 78.61287863090288\n",
      "  positive)\n",
      "C:\\Users\\jtd1g16\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\coordinate_descent.py:475: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Duality gap: 12487.608190337089, tolerance: 78.61287863090288\n",
      "  positive)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'alpha': 0.1, 'fit_intercept': True, 'l1_ratio': 0.0, 'max_iter': 1000, 'selection': 'random', 'tol': 0.001}\n",
      "Training Score:\t 0.684349499730894\n",
      "Testing Score:\t 0.6739856745936135\n",
      "ElasticNet run at: 2020-01-17 06:00:05.008927 . Saving to models/DRAGON_noVPAS/ElasticNet_lasso_17_01_2020_06_00_05.joblib\n"
     ]
    }
   ],
   "source": [
    "X_train, X_test = X_train_main[top_15_features_lasso], X_test_main[top_15_features_lasso]\n",
    "best_hp = grid_search_return_hp_dict(ElasticNet(), X_train, y_train, net_param_grid)\n",
    "\n",
    "net_lasso = ElasticNet(**best_hp)\n",
    "net_lasso.fit(X_train, y_train)\n",
    "exp_vs_calc_net_lasso = produce_exp_vs_pred_df('top_15_features_lasso', net_lasso, \"ElasticNet lasso features\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Scores and visualisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-17T06:00:16.590261Z",
     "start_time": "2020-01-17T06:00:16.554325Z"
    }
   },
   "outputs": [],
   "source": [
    "exp_vs_calc_df_list = []\n",
    "model_list = []\n",
    "for var in dir():\n",
    "    if isinstance(eval(var), pd.core.frame.DataFrame):\n",
    "        if var[0:3] == 'exp':\n",
    "            exp_vs_calc_df_list.append(var)\n",
    "            model_list.append(var[12:])\n",
    "exp_vs_calc_model_dict = dict(zip(model_list, exp_vs_calc_df_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-17T06:00:27.956342Z",
     "start_time": "2020-01-17T06:00:27.942319Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['test_scores.joblib']"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "joblib.dump(train_scores, \"train_scores.joblib\")\n",
    "joblib.dump(test_scores, \"test_scores.joblib\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-17T10:47:04.876735Z",
     "start_time": "2020-01-17T10:47:04.264566Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Training Scores'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'GradBoost, model-chosen features': 0.9136506916481263,\n",
       " 'GradBoost, f_reg features': 0.9317395172085381,\n",
       " 'GradBoost, Mutual Info': 0.9379608336062498,\n",
       " 'LASSO-features, GBR model': 0.996589735091017,\n",
       " 'Decision Tree, model-chosen features': 0.9141647290701933,\n",
       " 'Decision Tree, freg features': 0.824735879420048,\n",
       " 'Decision Tree, mutual info features': 0.8595461536744464,\n",
       " 'Decision Tree, lasso features': 0.8840999565633477,\n",
       " 'AdaBoost, model features': 0.9926354574810684,\n",
       " 'AdaBoost, freg features': 0.9979723730365371,\n",
       " 'AdaBoost, mutual info features': 0.9997045578005971,\n",
       " 'LASSO, model features': 0.7025515174084993,\n",
       " 'LASSO, f-reg features': 0.7247376293705728,\n",
       " 'LASSO, mutual info features': 0.6858987473879607,\n",
       " 'LASSO, lasso features': 0.6833898806906213,\n",
       " 'RandomForest, model features': 0.9774090544159082,\n",
       " 'RandomForest, freg features': 0.9721752491187455,\n",
       " 'RandomForest, mutual info features': 0.9729434173898541,\n",
       " 'RandomForest, lasso features': 0.9714625413023873,\n",
       " 'LARS model features': -5.118873363102049,\n",
       " 'LARS freg features': 0.412098008114794,\n",
       " 'LARS mutual info features': -4.03397309481404,\n",
       " 'LARS lasso features': 0.6395259500652293,\n",
       " 'ElasticNet model features': 0.7134538033619403,\n",
       " 'ElasticNet freg features': 0.7332180807485775,\n",
       " 'ElasticNet mutual info features': 0.6970302526384997,\n",
       " 'ElasticNet lasso features': 0.684349499730894}"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'Testing Scores'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'GradBoost, model-chosen features': 0.8532764923340206,\n",
       " 'GradBoost, f_reg features': 0.8411981283092483,\n",
       " 'GradBoost, Mutual Info': 0.8544833146624162,\n",
       " 'LASSO-features, GBR model': 0.8161460167193553,\n",
       " 'Decision Tree, model-chosen features': 0.7956771880980333,\n",
       " 'Decision Tree, freg features': 0.7707477970128914,\n",
       " 'Decision Tree, mutual info features': 0.7715041249308991,\n",
       " 'Decision Tree, lasso features': 0.7448102791928264,\n",
       " 'AdaBoost, model features': 0.8080655222178783,\n",
       " 'AdaBoost, freg features': 0.8433963130222173,\n",
       " 'AdaBoost, mutual info features': 0.8365182791277487,\n",
       " 'LASSO, model features': 0.7357933019197427,\n",
       " 'LASSO, f-reg features': 0.7171477393839785,\n",
       " 'LASSO, mutual info features': 0.6714929045146443,\n",
       " 'LASSO, lasso features': 0.6737594517258707,\n",
       " 'RandomForest, model features': 0.8582042794564964,\n",
       " 'RandomForest, freg features': 0.8401657504277648,\n",
       " 'RandomForest, mutual info features': 0.8415611230823128,\n",
       " 'RandomForest, lasso features': 0.810773797396149,\n",
       " 'LARS model features': -4.66823156501682,\n",
       " 'LARS freg features': 0.36890756929970014,\n",
       " 'LARS mutual info features': -3.6461882169232736,\n",
       " 'LARS lasso features': 0.6194334350344687,\n",
       " 'ElasticNet model features': 0.7453632055451128,\n",
       " 'ElasticNet freg features': 0.7301068054649074,\n",
       " 'ElasticNet mutual info features': 0.6787535164371025,\n",
       " 'ElasticNet lasso features': 0.6739856745936135}"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "66605d5a09e04ff3823de762bb6900af",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "interactive(children=(Dropdown(description='Model', options=('ada_freg', 'ada_lasso', 'ada_model', 'ada_mu', '…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(\"Training Scores\", train_scores, \"Testing Scores\", test_scores)\n",
    "mpld3.enable_notebook() #must enable\n",
    "@widgets.interact()\n",
    "\n",
    "def show_pred_vs_actual(Model = model_list):\n",
    "    pred_df = eval(exp_vs_calc_model_dict[Model])\n",
    "    model = eval(Model)\n",
    "    prediction_plot_scores(model, pred_df)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "notify_time": "30",
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "303.516px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}