final2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Final Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "import xgboost as xgb\n",
    "import pandas as pd\n",
    "from sklearn.linear_model import ElasticNetCV, LassoLarsCV\n",
    "from sklearn.ensemble import GradientBoostingRegressor\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.metrics import r2_score\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.svm import SVR\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "from sklearn.linear_model import SGDRegressor,Ridge,Lasso\n",
    "import phik\n",
    "from category_encoders import TargetEncoder\n",
    "from sklearn import preprocessing\n",
    "from sklearn.model_selection import GridSearchCV,RandomizedSearchCV\n",
    "import warnings\n",
    "import joblib\n",
    "from sklearn.ensemble import GradientBoostingRegressor,RandomForestRegressor\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import train_test_split,cross_val_score\n",
    "from mlxtend.regressor import StackingCVRegressor\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Reading train and test csv files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv('train.csv')\n",
    "test = pd.read_csv('test.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Target variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = train['y']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Corelation matrix of the train data using phik"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "interval columns not set, guessing: ['ID', 'y', 'X10', 'X11', 'X12', 'X13', 'X14', 'X15', 'X16', 'X17', 'X18', 'X19', 'X20', 'X21', 'X22', 'X23', 'X24', 'X26', 'X27', 'X28', 'X29', 'X30', 'X31', 'X32', 'X33', 'X34', 'X35', 'X36', 'X37', 'X38', 'X39', 'X40', 'X41', 'X42', 'X43', 'X44', 'X45', 'X46', 'X47', 'X48', 'X49', 'X50', 'X51', 'X52', 'X53', 'X54', 'X55', 'X56', 'X57', 'X58', 'X59', 'X60', 'X61', 'X62', 'X63', 'X64', 'X65', 'X66', 'X67', 'X68', 'X69', 'X70', 'X71', 'X73', 'X74', 'X75', 'X76', 'X77', 'X78', 'X79', 'X80', 'X81', 'X82', 'X83', 'X84', 'X85', 'X86', 'X87', 'X88', 'X89', 'X90', 'X91', 'X92', 'X93', 'X94', 'X95', 'X96', 'X97', 'X98', 'X99', 'X100', 'X101', 'X102', 'X103', 'X104', 'X105', 'X106', 'X107', 'X108', 'X109', 'X110', 'X111', 'X112', 'X113', 'X114', 'X115', 'X116', 'X117', 'X118', 'X119', 'X120', 'X122', 'X123', 'X124', 'X125', 'X126', 'X127', 'X128', 'X129', 'X130', 'X131', 'X132', 'X133', 'X134', 'X135', 'X136', 'X137', 'X138', 'X139', 'X140', 'X141', 'X142', 'X143', 'X144', 'X145', 'X146', 'X147', 'X148', 'X150', 'X151', 'X152', 'X153', 'X154', 'X155', 'X156', 'X157', 'X158', 'X159', 'X160', 'X161', 'X162', 'X163', 'X164', 'X165', 'X166', 'X167', 'X168', 'X169', 'X170', 'X171', 'X172', 'X173', 'X174', 'X175', 'X176', 'X177', 'X178', 'X179', 'X180', 'X181', 'X182', 'X183', 'X184', 'X185', 'X186', 'X187', 'X189', 'X190', 'X191', 'X192', 'X194', 'X195', 'X196', 'X197', 'X198', 'X199', 'X200', 'X201', 'X202', 'X203', 'X204', 'X205', 'X206', 'X207', 'X208', 'X209', 'X210', 'X211', 'X212', 'X213', 'X214', 'X215', 'X216', 'X217', 'X218', 'X219', 'X220', 'X221', 'X222', 'X223', 'X224', 'X225', 'X226', 'X227', 'X228', 'X229', 'X230', 'X231', 'X232', 'X233', 'X234', 'X235', 'X236', 'X237', 'X238', 'X239', 'X240', 'X241', 'X242', 'X243', 'X244', 'X245', 'X246', 'X247', 'X248', 'X249', 'X250', 'X251', 'X252', 'X253', 'X254', 'X255', 'X256', 'X257', 'X258', 'X259', 'X260', 'X261', 'X262', 'X263', 'X264', 'X265', 'X266', 'X267', 'X268', 'X269', 'X270', 'X271', 'X272', 'X273', 'X274', 'X275', 'X276', 'X277', 'X278', 'X279', 'X280', 'X281', 'X282', 'X283', 'X284', 'X285', 'X286', 'X287', 'X288', 'X289', 'X290', 'X291', 'X292', 'X293', 'X294', 'X295', 'X296', 'X297', 'X298', 'X299', 'X300', 'X301', 'X302', 'X304', 'X305', 'X306', 'X307', 'X308', 'X309', 'X310', 'X311', 'X312', 'X313', 'X314', 'X315', 'X316', 'X317', 'X318', 'X319', 'X320', 'X321', 'X322', 'X323', 'X324', 'X325', 'X326', 'X327', 'X328', 'X329', 'X330', 'X331', 'X332', 'X333', 'X334', 'X335', 'X336', 'X337', 'X338', 'X339', 'X340', 'X341', 'X342', 'X343', 'X344', 'X345', 'X346', 'X347', 'X348', 'X349', 'X350', 'X351', 'X352', 'X353', 'X354', 'X355', 'X356', 'X357', 'X358', 'X359', 'X360', 'X361', 'X362', 'X363', 'X364', 'X365', 'X366', 'X367', 'X368', 'X369', 'X370', 'X371', 'X372', 'X373', 'X374', 'X375', 'X376', 'X377', 'X378', 'X379', 'X380', 'X382', 'X383', 'X384', 'X385']\n"
     ]
    }
   ],
   "source": [
    "corelation_mat = train.phik_matrix()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Label Encoding  categorical features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "##Categorical columns\n",
    "cat_columns=[]\n",
    "for i in train.columns:\n",
    "    if train[i].dtype =='O' or train[i].dtype =='float64':\n",
    "        cat_columns.append(i)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Label encoding features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "##Categorical train data\n",
    "##Categorical train data\n",
    "cat_train = train[cat_columns]\n",
    "##Categorical test data\n",
    "cat_columns.remove('y')\n",
    "cat_test = test[cat_columns]\n",
    "\n",
    "##Label encoding categorical features\n",
    "for i in cat_columns:\n",
    "    en = LabelEncoder()\n",
    "    en.fit(list(train[i].values)+list(test[i].values))\n",
    "    joblib.dump(en,'final1/en_'+str(i)+'.pkl')\n",
    "#     cat_test[i] = en.transform(test[i])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Binary data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "##Binary data\n",
    "binary_columns=[]\n",
    "for i in train.columns:\n",
    "    if train[i].dtype =='int64':\n",
    "        binary_columns.append(i)\n",
    "binary_train = train[binary_columns]\n",
    "binary_test = test[binary_columns]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Merging encoded and Binary Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "X1_train = cat_train.drop('y',axis=1)\n",
    "X1_train = pd.concat((cat_train,binary_train),axis=1)\n",
    "X1_test = pd.concat((cat_test,binary_test),axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "XGB on target values less than 200 and top 260 features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 272,
   "metadata": {},
   "outputs": [],
   "source": [
    "##Index of data with y values less than 175\n",
    "outlier_index = train[train['y']<175].index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 275,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['final/outlier_index.npz']"
      ]
     },
     "execution_count": 275,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "##saving the index as a array\n",
    "joblib.dump(outlier_index,'final/outlier_index.npz')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Final data for training with top features and target values less than 175"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_260_features = corelation_mat['y'].sort_values(ascending=False)[0:260]\n",
    "top_260_features=top_260_features.drop('y')\n",
    "X2_test = X1_test[top_260_features.index]\n",
    "X2_train = X1_train.iloc[train[train['y']<175].index]\n",
    "y2 = y[train[train['y']<175].index]\n",
    "X2_train = X2_train[top_260_features.index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['final/top_features.npz']"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "##Saving top features as a array\n",
    "joblib.dump(top_features.index,'final/top_features.npz')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Parameters for Gridsearch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {},
   "outputs": [],
   "source": [
    "params1={}\n",
    "params1['eta']=[0.06,0.04,0.02]\n",
    "params1['n_estimators']=[250,500,750,1000]\n",
    "params1['max_depth']=[2,4,6,8]\n",
    "# params1['objective'] = ['reg:linear']\n",
    "params1['reg_lambda']=[0,1,2]\n",
    "# params1['interaction_constraints']='[[314,315],[229,339]]'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 252,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 2 folds for each of 144 candidates, totalling 288 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  48 tasks      | elapsed:   43.4s\n",
      "[Parallel(n_jobs=-1)]: Done 138 tasks      | elapsed:  2.9min\n",
      "[Parallel(n_jobs=-1)]: Done 264 tasks      | elapsed:  6.1min\n",
      "[Parallel(n_jobs=-1)]: Done 288 out of 288 | elapsed:  7.0min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=2,\n",
       "             estimator=XGBRegressor(base_score=None, booster=None,\n",
       "                                    colsample_bylevel=None,\n",
       "                                    colsample_bynode=None,\n",
       "                                    colsample_bytree=None, gamma=None,\n",
       "                                    gpu_id=None, importance_type='gain',\n",
       "                                    interaction_constraints=None,\n",
       "                                    learning_rate=None, max_delta_step=None,\n",
       "                                    max_depth=None, min_child_weight=None,\n",
       "                                    missing=nan, monotone_constraints=None,\n",
       "                                    n_estimators=100, n_jobs=None,\n",
       "                                    num_parallel_tree=None, random_state=None,\n",
       "                                    reg_alpha=None, reg_lambda=None,\n",
       "                                    scale_pos_weight=None, subsample=None,\n",
       "                                    tree_method=None, validate_parameters=None,\n",
       "                                    verbosity=None),\n",
       "             n_jobs=-1,\n",
       "             param_grid={'eta': [0.06, 0.04, 0.02], 'max_depth': [2, 4, 6, 8],\n",
       "                         'n_estimators': [250, 500, 750, 1000],\n",
       "                         'reg_lambda': [0, 1, 2]},\n",
       "             scoring='r2', verbose=5)"
      ]
     },
     "execution_count": 252,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf = xgb.XGBRegressor(objective='reg:squarederror',)\n",
    "grid3 = GridSearchCV(clf,params1,scoring='r2',cv=2,n_jobs=-1,verbose=5)\n",
    "grid3.fit(X2_train.values,y2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Saving the best model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 254,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['final/best_model1.pkl']"
      ]
     },
     "execution_count": 254,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "joblib.dump(grid3,'final/best_model1.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Submitting to kaggle "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [],
   "source": [
    "best2 = grid3.best_estimator_\n",
    "pred = best2.predict(X2_test)\n",
    "prediction = pd.DataFrame()\n",
    "prediction['ID'] = test['ID']\n",
    "prediction['y'] = pred\n",
    "prediction.to_csv('stacked1/x1.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x18639e3dc48>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAzUAAAB6CAYAAACGCn3IAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAA35klEQVR4nO3dd3Rc133o++8+ZTp6B0iABSTBIlaYoqhiFRfKsS25O3Fs2XEipfg5TpybOO+ue/Nyb3ydrOTGiR3HjmzLliO5W1Zx1CUWSSwiWMFOkCAIguh9MO2U/f6YIQmSICU2kaB+Hy2umTn1d85sHZzf7HKU1hohhBBCCCGEmKyMqx2AEEIIIYQQQlwKSWqEEEIIIYQQk5okNUIIIYQQQohJTZIaIYQQQgghxKQmSY0QQgghhBBiUpOkRgghhBBCCDGpXZGkRim1Sim1XynVopT6ypXYhxBCCCGEEEIAqMv9nBqllAkcAN4NHAM2A7+ttd5zWXckhBBCCCGEEFyZmprlQIvW+rDWOgP8FLjnCuxHCCGEEEIIIa5IUlMDtI/7fCw3TQghhBBCCCEuO+tq7VgpdT9wP0A0Gl3W0NBwtUIRQgghhBBCXOOOHDlCX1+fmmjelUhqOoCp4z5PyU07jdb6QeBBgMbGRt3U1HQFQhFCCCGEEEJcDxobG88570o0P9sMzFJKTVdKBYBPAk9egf0IIYQQQgghxOWvqdFau0qpLwDPASbwkNZ69+XejxBCCCGEEELAFepTo7V+Gnj6SmxbCCGEEEIIIca7Ig/fFEIIIYQQQoi3ylUb/UwIIYQQQoi3k3h8jLb24yRSKdD6Ta+nlKKqoozqqgqUmnDwr7c9SWqEEEIIIYR4CxxqPcrg8AjBYADjApKTTMbhSHsH1VUVVzC6yU2SGiGEEEIIId4CiWSScCjIjOm1hILBN71e+7FOBoeGr2Bkk58kNUIIIYQQQrxFDNMkGgkTCYff9DqBgNyyvxEZKEAIIYQQQggxqUlSI4QQQgghhJjUpC5LCCGEEBdB43kunqcxTAvLfAt+J/U9Mq4HysCyLAwZBOrK0hrXdfG1xrRszLfghGvfw3E9UCa2ZXJmX3qtPdyMi2+YBCzzrJHAtPbxXBdfKyzbuqDO+GJye1skNVprejq7+NGBEQZCMf7qxmoKpZALIYQQF0VrTe/BV3n6pR0MpV2CBVO46c67uKG2CHOCP6/Dbdt5bu1GBhLjJhZN5wPvvo2a4jB6rJ3fPPYiHWPpcQtEufGDH2ZJdRS0ZrR7P+teXEdLXwLMABUNK3n/7QuJ2acnU9oZYfsrL9PUOkzDLe/n1jklp+b5Hn2HtrJmw0504SLe+8FGCi7vqbluaN/h4KanWbPlCAkXYtVzueuuW5lWHD4r0QCHjh2v8uLr+0l6p6ZGaxfywTtupCBs4nfv5Ie/2UTGObWAMqq5+767qQ3agCYxcIRXX1rD3o5hsIJMXXgH77t5DiFTAZrU4BFWP7+aA53DaBVgyoJbee9tC8mzczF7o2x7/r/YcKALR5uU1y/mzjtWUBG1kbu+69/13/zMTbFpyz4+9F8H+cdtHTy0r4+xqx2TEEIIMWn5pI6s45++/jBbOh2mTp/C4I5n+dev/5B9p2UtJ7j0tm5n9XObGfDHPZdD+2hyn/sP8czqNRzqS5+2pvZzW0ge5af/+s880dRN5cxZVIbirP3pN/nR6sM445fXPt0HN/DYT3/K08+9SHP7yKmZXoLdq/+Tr339uzz1m2dZt+GA3A+ci/Y4vunn/NODT9KWjjKtJp8DL/6Yf/ruk/SlvQmWT3Fkx0Zefu0AqfGTfQ2573igbRsvr1tP9/iTrjW5rxh3rJenf/hNfrZ6L5HyKURTHfzXw//Cj9e14miNM9rJkw9/h8fWHaS0voHKwAjP//RBfvHyHlIeaG+M7U89xLd/uZrRYAV1pZpXfvUQ3/n5WkYzPuL6d33U1KQSrGsbJhOJsbI6jwgOB48P0pwwuCOq+fGRBHfPr8bt6OaH57qCaY/+vmHWdCXp96EqL8ydtUVELYV2Xdo6B1g/mCGuDeqLY9xSE2Wwf4iNvS5L6sqpjSrQHsd6Btjcr1k5u4wKS34XEEIIcZ1xhtnwwksMhubwxS99gQVFJkPzK/nGP3+HX69tpeHD8zFPW8EjOZZER+fzyfvvZ/q4Zmonmg4lx0bxwzHu/Nj9rJodGrdudr7u7WCgdCn3ffBj3FRfjHLfQbD/H1i3fjv976mnMre0n+7lxceexp2+kNqWttPjHmnl1W2jvPNTn2b0+SfYnLncJ+b6oRPtPPfMBqINq/jCH3+EyqDHouoI//fBZ3hh77v4nSUlp6/gOYzGU4Tqb+P+B+5h/JheJ77jsZFhosUzuPczD7Cw9LQFAOjb+wIv7DH58Bf+jFWLKjHcO6l75FEOdR5hKD0do7+FXbtSvO//+UvuXVgN3EHxN/6Mx7Y0seqmORSNHuT5V7ZSeesf88XPLCesHeoKvs03fv4MO9+9glumxK7oORNX3/WR1BgGnS3t/O1AkO/ePYeFRoJvrtlHW/E03n1HJX9+x1xqQh7/3t094epa+7Qdbef/W9fBK6MeBaZi0Df53Rvr+ctFZXTsOsQfbu4hbphEfY9+HeDLd8znDmeIr647zsdujfAX82PoVJonN7XwL4MR1swue4tPghBCCHHl+fE+9rb3kD/3PcwpzjbrKaytZ8H0al7avpfBD89n/D0rrsPgwCCp4mpSezfz/NEBCusXsXhmFQFToZRmsL8XpSOYY62sfr4No3gGSxbMID9kAQqrajH3f34BkWgUfB83NcZQ0sHKixLI7UbrDEfWP8Gm7gI+dt+tvHjs2OmBx+r40Gc/S2lJgidXA5LUnFOi5xiHBoeZ+c4bqQgbKAymNixgRsXL7N19hPSSEsY/YUWnkvSPDhEqi9K2ZS1HulzqlixlVlURFqBUir6uQQJGCYmuLTzf1EtR3Q0smFVF2DJAubTu3EGobgGzKwx2vPoi/ZQw++5P0BiOELPBL5vLp/70C5TVl2UTId9EGRCwbQxDM9BzlO5um9vvbyBiKMBmxrxG6vI20nx4WJKat4HrI6kJhFi1soYnfn6If9/Zz8f847yQifHdlRXEwjZ5kQAk4udc3U2nePS1o6x2C3j8d+cyJ+Sx7pV9fG1/N79Vk8/WfT0MRMv5yd0zmGEm+OXWY/SMJaiakc+N4eO81N7HfXNiWGNx1h5Ps3zeTCrMc+5OCCGEmLRSY3GGxkYoriw9mVAoO0x5YRTVPsRZDdB8Hx8F3Rv54SMHMFWGwcd+RfWKT/CHv3MnZVFF2tGoVC+/+cmPiFkwMvALflFzE3/+hU8zsziECkQoCgDdzXznZ89y5PAhujJV/MkfvJMiAO0zfGgTjzy5g4b3f5EbZ6R46cw47BgVpcDZEYozxIeHSCYSlJWVn+ynYMXyKQuG6RocIQ2nJTW+5+ObBsNbnuKHR2IoN0nvr39F44ce4L5VC4lYDhkvQKJ/B798tIOAdhjo/QVlyz/Mlz93N0XBMfq6kvgDR3jk+98kM+aRiQ/wM13DZ//si9xUoLCjZcy5IfuDsfbSdO96nqd2Blj+6eUURzWHR/uJjxRTXpHrYIMiEi0gGgsyOHzue0Bx/bg+khogP7+CBxb288COVvb5mvcta2BpXnCCzmxnS3kDbOr1uXtlDfNiAYIGvOsd9Uwb8ajKN+ktCtDX0s/XtxjcXBZl5YIZzCoJYSiHe6vCPHB8lJZBl+LOTjZ7Ib4+O4YhXdKEEEJchzQ6201i/B9YZWBaBif6T5zGDNFwy/u5fw7MnjuHwkCKHS8+wnd+9iyvL1/A+xZWUNJwB5+/fwHFM+cztdCmbevTfPt7j/PrTbfxF3fPObUtO0xZRRUKHzrTHNpziIaqGwg7w7z63FMMla7kgdvqCaZ2XenTcF3TOtsX5rSRxQwjN/rZBN9xqJhb3vfbzHp3KfPnTCXoDvD8I//GL37zLDcvr+eG8gAzbrmXB+amqW1ooDSQYdvzj/C9Xz3B2ltv5t752YEJjg/00viBP+RjN88h072T//y3b/PUU2uZ9Yf3UBE6EZtH197XeOihp6hY+XFWLZ9OQHmgNRp12t2XYSiUYUwUsbgOXTcDBSjDoL6sgBLfoVVbzC8OEXqTtSWeTjPoKPJj6uQ12syLMasqn/yQxZ0r5vKthXl0tffyD+sP8aHHd/D1ncN4BFgyv5DisVE2D46y4WCcotJSFsRCSE4jhBDiehQMhgmHw4wOx0928tZOkv7BBDoUOVl7c4I2bMqmNrB82XxKYgGsQB4Ny++iPtJDR18cjUGsvJaFjY1MK4th2UGmz29k0YxCWo9km437novjuPiFM7jnI5/gs7//APcu8HnumV+zrzNF/76XeWJLksWN9SS7j9HW0UPKcRgd6KJrMIGWu9oLEo7GCARDjI6OnEwIdDxOTyqFGQljn7G8EQgzddYS3nHDdGJBm0C0nMbbVlI61E13Kg3aprSunqXLFlKZH8IK5bGocQVTSx0OHx8GDEzLpKx2HisXzyUctCmsXczty6cw1NvK0In+0Npj+OgWfvCdn5BsWMXnP3obpSEDMAhH8glH4oyOnBjIQJNIxEmMjRENBRHXv+smqXHSY/xixzEGIkV8IOTy8J5ujqW9c2bn2nfo748zltEEjAJmx3x2tSXIeLlROjq6+O7WDo6MjNE+lGHG3Jk8dt9NNH1sNneFMvxgTwftQF5pFb+V5/H8vlYe6fa5fVY+NVGppxFCCHF9MvNLmV1ZzmBzE61xF7Qm0dXOnmM9lC2ZSwngpccYGh7D1aDTcba89Et+tXY/js6Od5aJ9zCYDBIKWShStG5+kV89vobetA9onGScoZExwpHsz/N9+9by6C9/w/7uFKZlEwxHKIxFcZ0kaccjPjhESXmUfWsf58H/+A7f+f5jtPQPsOfln/ObzW24MvjVBckrr6amoID9W7YylNGgPbqPHOToQIrZC6YTApzECEPxFD6QHu7m5Sce5aXdXejcf2PDPYwaQUKGAe4QW194kidfbibhAVozNjpMIuHmEo586udPITUyyOBoEjTozChHO4awA1FsG/Bd+g9t5Dv/93ukFvwWD3zqA1TETqRXBsVltZSVD9G0+SgZnR2+u6N1N50j5dxQX3KuQ33LGYaB4zj09g3S2d37pv+NjSWvdujXvOuj+ZnWHG05xn90w+dvq2OV0cvn13Ty6IFSvnJDARO1QetvPc4fbOzl9mUz+YOZBbx3ZoT/dqCd/xVxuD2m+XlzJ9uCxdw0q5D1rx/gB26EL80rocwdoy2lKSwPUghYdpBbZkX48esjxIMx/rSikJBkNEIIIa5TKljMijtX8uI/P8Z/PpTPbYvKOLTxOXaNVvJHN9djk6T5+Uf51UGTD3/iE9xQrnAH23nupdcZHXwvDcWwdd1zDJQtYPGMcpSyiOhBVv/mZVqHhriloZhDTS+xtbeQT3xmOgDhMOx49pc0txzjlqUzMce6aFq7m5Kpq5hWFqa8/CN8ecG4wZ07NvDX//Y4c9/zaT6ycgbWdfMT7lvDKKjjvbcv4u9//CQ/KMywtNpj8wvP0l+4jLvmlaLoZ/XDD7E9UM/vfOwDVFow0raTZ19tY/i976Rc9bH6ufVEl9xJfWEMzAxW/DBPPvMqnYOrWFLu0LT6BToD8/nU3BJAMWXZe5j5/L/xsx89zNDti0i2bODprUPc8PGbqYppxvoP8KuHHmZbv8k7iwz2bHiZPQB5FaxYPJ/86lm8s7GB7z//Ix417mBaYISXn11HcP77WVQVfoMjfuuUlBRxtP04h1rb3njhM4TDoTde6G3s+khqUkM83ppkRcNUPllfQJUf4I9mJ3istYejcwuoswHDoLQgylwzW20aUDCmwVcay7L44IoGHOMI/364mzVaEYsV8K3b6llQYFN1ax2HXj3ON7d1oFGUlpbxrdvrKACUMrmhro7bjrYyUFLKLVVnVrwLIYQQ1xOTknmr+JPfd/jJ46/wVCtY0Rp+548+TuOUPMDBNh3SGQ/DMFFWiOXvvw9X/ZynXnuO3Rryq27gjz//IeZXRFEoqt5xL1/MaH761Hp+vcfHLpjCJ//449w+OzuOWqzuNr70JZ9fP/kC615oQSuT0hvu5c8/uYrqqIGhCimLjgvRrWbatGlMqamgKHZm0yObgopqapJFZzWjEjkqRP0dv8v96Ud4Ys1LPLFVEa5cxhc+8RGmxEzAwlQpHMfHMAysaAXv/+wX8H/8KBuffwJfmVTOez+f+8i7qIhaKGWx8IOf53P+Izy96VkOaUVezSL+6L57aCjJfguRykXc/4Xf47HHnuHZXx/CJ8xNn/pTPnpHAyED+hN9DI6FqCg1ObR5HYdPxFq1mHlz5lBQUsjKjz1A2n+IZ9Y+y05lUDpnFZ/++PsoCV47ozdNr51KbU31Ra2r5MHx56X0NdDQtLGxUTc1Nb2l+9Ra42VcVNA+OZ6+1ppMMkXSM8iLBDDHPxbZ94iPpfENk0g4KL/6CCGEeJvTeE6GdMbFCkUIjPubqbWH64Jtj7uZ1BrXSZN2FeFwAGOCGzTfdUhnHKxgGNuc6AbOI5VIo02bYNC+ftrQX7N8nHQGx9MEQiEsY/x37OJ5Btb4GyKtcTJJMtoiErQnuAnXuJk0GReC4SDmBGVAa490Mo2yAwRs68Kb82ufTDqF45vnLGdi8mpsbKSpqWnCL/X6qKm5CEoprKB91rRgJMyE3ckMk1he5C2JTQghhLj2KUw7SMQ++6+mUma2H8TpE7ECIazzNGgwLJuwdb76E5NQRP4Wv3UM7GBowhotpSysM+8ilcIORs5TA/bGZUCpS/yOlUFgggErxPVPfuQQQgghhBBCTGqS1AghhBBCCCEmNUlqhBBCCCGEEJOaJDVCCCGEEEKISU2SGiGEEEIIIcSkJkmNEEIIIYQQYlKTpEYIIYQQQggxqV1SUqOUOqKUalZKbVdKNeWmFSulXlBKHcy9Fl2eUM9P6+xDwJKJFJ7W4LskkwnGxsZO/Uum8f3xDxvVuJkMjutzDTyD9DLyyaTSeBd5TG7GoS/p4o47V77vMpRwyXia+GicIwMp3Am3r3Fch+GUd9Y51W6G1s4hhjJ+9vvysq9n8hyX3oRD2jtjfd9nOJFmJHPu70trj5Hcur7jMJB28S/s8N8832M4mSGtNZ7j0p9yr9SehBBCCCHEeVyOmpo7tNaLtdaNuc9fAV7SWs8CXsp9vsJ8EgPHad64msd/tYZO14NEH9ubXmfDhg1s2LCBV9e+xOMvbmAo6YxbL03b9s3s7Ri98iG+hbQeZPsL6+lKXtz6xzbv5saf7WZtX+Zk8jAYb+WLv2iheVDTvOswX98+yHBmorV99rYc4H++3E/yjKTE7+vlHzcc5fXeDI4T56WdxzkyenYi0NHcwopHtvH4sTQncxetGRzo5Q9+tJH/vSPOufO1Hv7usd1sGoCR5oN8em07wxdzEt6M4QH+x2PNbHI1Pfvb+OiLhydM0oQQQgghxJV15rNgL4d7gNtz7x8G1gB/dQX2M06ctv3HCBSWEYsMgQai5bzjptKTN5kDR3ayqydAMDDBIWuN66RwPUUgGMBQiuxGNG7GwdMaKxDEVID2cRwPM2BjKoX2HDKewrZNfDeDxsBLJ3DtPKIBhe9myLg+ph3ANg2UUuN2q3Pbc/A02HYQ0wClQGvwnDSOrwjYFp7rYdoBDJVdJ5Nx0MrAtm0MxWnbJRc9vkc65aCVSSBgobRPxvGwbBvDUCjt42QcsGxs8/T8tndoiH/cdozld00nzzLGbVWxcO5Uqt0gBQGN1j5joymSLkSiYSKBE6dOk0klGUt5RArCRCwDo7yEv1oWorDMZnCgm5/s7GZVOEx5uJiofXr8w8kkLx/r531TqsgzFRpoPXScdXHNZ3Q2Fq0hGR8jnoZIfpiobZw69vHnQWt83yc+moBQhLyggXYyDIxmMAIB8qIBLAXpRArHsDG9NGNpyCuKEFSQLQ2axGiSMRei0RCRgIHKbXv8vryMw0jKpzA/iKFUbr8pgpEQQVtaewohhBBCXAmXmtRo4HmllAb+Q2v9IFChte7Mze8CKi5xH29ClPply7DS3Rw7PJKdpAys3M2474xwtL2XyoabiVhn3lj6DB7bzboDY7hOEi82k1tWziemUrQ2b+Fg1xim4eMGi1mydDEV5jAbtx5j/o0LKQlYOENHeH23x9Jb59C391X29hiYXpopC2+mItXCxr1d2AETxwmw4KabmVpoc+L2XXspWrZtpKU3g21pHLuIxsbFlEWg8+A2th0axLZN7IIigok40xtvp8waYve2LRwf8TF8F7OoluVL5pIfMk87Ku0mOdS8kT3xBI7jUly3gEUziti7tYnozKXMrsjHT3axYe1+Ztx6O1Ni49dWLK/JY+ZoP/+8vZi/WjK+BaHmwIFj/Nos48v5Js+v28X3+yDPBNeM8LV3TQdgePgYf/6MwaB26U2b/N2qhdxaMsCDGwe4590We5v7eG0kydHtxygtKOCuqtOLYlVxmEx7Hy3zS1lcGIDMEE+1JlhcFqYI0G6aV5pa+D+HMxSEYSCh+aNbFvChugmKh++wubmF77YZfPn2GRT1dPN36ztot228lENVZRX/ekcJG7fv4z/7TFTaYxAPZUX4yjvrWZrn89ymfXzrmEeeDT1p+Mv3LODuCf7v8YeG+IvV7Xzm7sXcXmDS39fKV9ak+eK7Z7GoSJIaIYQQQogr4VKTmlu01h1KqXLgBaXUvvEztdY6l/CcRSl1P3A/QG1t7SWGYRIIAhM2h/IYPrqPfquWeeV5nFGhAaRJqjrueFc9dqaf9Ws30j08G8s5xMG+AMveuYKSoEd780Z2HeqkfFYI1z3VX0RrH9f10YCfSaPy67l5aT1h4jSv7aZ8wS001kYY7mxjyM0A9qld+w5Wfi0rF9RRYKVp3rCRY/1j5Dtx9nUkWHjzHUwtsBls3cLaZoc6oL9tNz2qhlvvmkNYx9m7aSP7OqtZPr34tKNyvSR20TJuurEcb7idDa8foq+mkdI8m7aeQerLo4x2HWM0r5ry6NlnzS4o5A/nGnx1XSsvVIdYMS7p8TyPJD5JJ86aToffu30+Hy4L8EpLL2mdrdk4mjH5xvsWsCCS5j+e3sUvjwxwa4km6fh4uoCP3lzFxs4O7rl11lkJDUBZXoT5xhgvdqRYXGDTfaiXjWaYVVM1KcDP+IxECvjaeypZWqZ4+cWdfKu1m1V19ukb8jJs3XOIRw5rfu/WOubkWbQNwm2LZ/KB+kIy7R18aG03B3UxjuezN2nz499awLRQmh8938yjeweZtTQflV/C/7m7nEUxl5+v3s2vd49y96IJzltJjDvyNL85MMJt7yigZfcw6bIy6vLtsxcWQgghhBCXxSX9dKy17si99gC/BpYD3UqpKoDca8851n1Qa92otW4sKyu7lDDOFyF+YoC9LcPUNUwnMmEKF6SiuoyAUqhAEUV5GtfzGBsewCqfSnHYwjADlFWUkBgYJHW+LhNWkOKSUsKWgTIj1NSV0LnzNdas384ARVQVRU5bXFkRSouDHN+zmdXrXuNgRz9pxyOVGsWKlVEWDYIyyauuJD97OIwOD1FcPYWobWAG8qgsj9DTN3R2KIEIlVVl2IYilFdMWRCGk1BWWc5YXy/xTILjx5PU1FUy8e22orSmmvunGfxsy3E6k2d3tw/aEW4qt/jeK/v5q1fasPILmBPLnuQ5peVMj5kQiHJDQYCEe2Gd6M1QiFV1RazZ38lwKs5TrUMsmzaF2lB2vhEOsrI8wNbmAzzw+C7+7sgYCccjfdpWNHuO9vIXm4Z457JaVpaFUUpRVZpPdaKf//3sTj6/4TgH4y4nelUtqqxgRp6BYQVpnBUh1efgqgBLSmy2bT3I7z25n28cSXI8c44OS2aY99TG2NfRS3tyiJd7PO6YWk6+OfHiQgghhBDi0l10UqOUiiql8k68B94D7AKeBO7LLXYf8MSlBnmxtPboaW8lnl/DzJIgZ1XSAGBgGifmqJM1OYZhoL1TI2f5no8ycv0oPBc3V1WjXZeT/eGVQhkqu4wyKZq+lHffcSMzy4K073yFTft7Gd933hk4xNrX9uEXTmVpYyMNtWWYgFIGnpPtywOgPe/kespQeJ6X6zei8T0f0zj7jllrD8fNRq99H8fXGIYiVFxBtR6g5WgPvTpETUmECaqvcodjc+uiepb6g3xzd5wzb+NNO8Qn3rmAh26ZwpKgw1dX7+anbSkAAsoidyZyfZQukDKYX1fE7NQQ32vq57khm08siJwssMOdA3zxpVaOh6J8YUkd/2tGDOPM3WjIi0X4VE2IH28+yp4RB+15PLF2N19rz3D7jEr+dWk5DZFT/xuMuc7J7zzjeHhKMTzSwX9/4SgdwTz+7B1T+G8zouSd55BK5paxKDHKs6/2sN6I8t7pARk7XQghhBDiCrqUe60K4FWl1A7gdeC/tNbPAn8PvFspdRB4V+7zVeFn4hztHGFKbQ22cWGHGi2pRPe00jWcwsuMcbS9h8KKckK2TcQd4ujxIRKj/RxuPUZmorG4MnFadu2ixwlTM7OB+sooY6OJ05IanU5CrJSptdWEjQyDA8P4KCJ5JQRSPexp7WRkpJ+WvYcY0dl2XUVlFQwebWUo4eLE+zhyPMWUquKzdu+mkxw72k7S9RntO05X2qQ4ZoMZY2pthIPNe1CRIvKC1jmSPUAp7FiEzzaW0384TssZs5Ppfh7ccIyeaBEfX1LDzRHN8ZE3N4CyUkFMPHpGEiQmHhsaVVjIJ6ptvrWrg8LqMmbYp5K3lOszpmw+tKCGOUWKTQNp/DM3oxRTy4r43G2zuNce4+83HKcr6TOQ1txQU8Zts0rojSfpS5+KeU9nDxt706QSwzy/N0lZtY1lpnCsMO+dW8XMPJOdA2kS56mxM40ibiz1ePBgL9NrKqiRjEYIIYQQ4oq66D41WuvDwFm9CrTW/cBdlxLURTMsonkRrFzNQHqoi3Somrll+Ux8564IhMK4J0alUhAMx1CmQbCojoUzR2l+7QW2+wb5ldNZMqMMgoq5C6ayYfsGXjJi1NWVU6yCGIAdjBA6ceNtRygvsXh9w4tsdzVWuIJlK2oIjNu7XTaDutb1rH2mnUA4QjSvlEjAxIoWs+wdi9nTvI9NHRZVdeUUdnQBUFgzj1nD29n48jN4yqa8bi6zzuoUY5JfXIGre1n73C4cI8r0eYspDdsopcifOo2CbX0UV1QSPKt6A+xwkCnYZI/EoLiqhv+5KM7fdgQJmQodDlFuWITNPKZbx/kfz+4kjmJ6STn/sz6E0xOkPGaePOWhSJDKgIVCU54fJGRCyM7n/TMifH3LEQy7kN+tP1UUrVCAqVEbsFmwuIzbu/q4u76AiGkQC4cosQzKy6J8qtriS09tB9Pipmh2HROL8vwgYROM3HbsSJRP3zab/lfaWN1dxLsWlfG1zW28d/9RFsRMphRHiABjwLKiCD9d08xfZzSzKyr473NLqDZj3FW+j79+ZgcqYNNghJgWtcEwqSgIEVFgBW1qY9kzppRi/uxCoq2D3NMQ4xyFTwghhBBCXCbqWniuRmNjo25qarraYUzI9xxcX2Hb42s0NFprtFYYEyQFpxbT+L6L44FtWxM3w9I+juuiTBsrty3fSXC8o49YRRWFYYvEwH5e3dDD4jtuozyS3b/rOGhlYZ81mttpG88uZ1i5IZs1vueRHGhh0/ZhFt22nJLgpd5wa9KJNGlfEY0FubCuI5p02iMQMM8akvpNra016bEUnmUTCZ2nxmkCXiY7bHMkL5jL7JM8t24Pr9jz+JulJkkH8qL2uLh8kvE0vh0kGjz/Oc8k0zTtPcQPusJ89a5plJ93eSGEEEII8WY0NjbS1NQ04S3flXhOzXXFMG0CZ92pK5RS5+qKMm4xhWHaBM93p68MbDtw+iSlcIeOsn7vHmzbwk1nqJy7nKLQqf1bZ6xzjo2fvpzv0tWyg+YDPdQsuYXCS05osvsIRkIEL3bd4MUXQaUUoVj4otY1A0HyTzuFCtMwsA2FHQxgn3VABuE3sy8/zq83tfKj4y6fubGO4oAkNEIIIYQQV5rU1FyLdPahlk46RcYDKxAgcK6angvcrq99fB8M07j07V1XNJ6n8ZXCPl/t2xtuRuP6GleDbSpMOcdCCCGEEJeF1NRMNkqhlEkgHOXN1MdcyHYNZXKBYya8TShMU11g87mJNqOwTCX/YwkhhBBCvIXk9lYIIYQQQggxqUlSI4QQQgghhJjUJKkRQgghhBBCTGqS1AghhBBCCCEmNUlqhBBCCCGEEJOaJDVCCCGEEEKISU2SGiGEEEIIIcSkJkmNEEIIIYQQYlKTpEYIIYQQQggxqUlSI4QQQgghhJjUJKmZVDRaa3zfR2uNvqhNnFp/Uprs8QshhBBCiMvu7ZnU+BkGe7roj2cmnu8k6O7qJJ65fDfOyaFuuvtGcS9lIxrc5CD7duyhZyx9UZvwkkMc2L2TjmHnUiJ5Q1qn6T3exUjyjfaj0drH87w3lah4iX727Wqma9RDu0l6u7pJXNlDEUIIIYQQ17i3Z1LjjLKv6TWaj8Unnh/vYsNrr3B0+PLtsvfAJjZtbyN5idvx/QyjI6OkXf+i1te+w9jICMmLXP9N70cPsW3deg71jL3hspmxPpp37SeeeuOUz/cyxEdGSLk+pHppWr+JrnN8jUIIIYQQ4u3ButoBXC5aazwnQzqTwdcKKxAkGLBQSmWbLHkO6VQaTyts5U+wbppU2gHDJOid54Zfazw3u6yvFVYwRMg2AZ90MoMZDGKbBqBxM2kcbREOnjjNGi+dJJ5xwbAIhoJYhgF4ZJJplG3jZdK4WhEIhrANn3QqjeuDHQwTtA3scAnzlkYJhkOARvseqVQK19Mo0yYcCmIa6uQxpTMOPgZ2IEjQNrHCRcxesJhALHjy2N1MirTjojEIBEMELAOlwE2ncLAwtUPG8VCmTTC3fTXhecmQSmfwMQkE/Anm584bCjsQImib4DuMDvfRdrSd4soqLDMvez61TyadIuP6qHHnyo6WMveGGMGYzWkZotb4vkcmncbxfAzTJhQKYkwUqxBCCCGEuK5cN0lNZqST7dt20TuaQRkKM1jAvKXLmFoUQjsj7Nu6lcPdIxh2gFhBHmNJh8i4dbds2UFf3MUOhCjOt89RE6JJx3vYt30nHXEXXAdVUEXjsqWURgbZ8tLrlCy/g4aKMODQuXs9u/x67l5WC0B6tIfm13sYHU3iuIqSKQ0sXTiDsDnElhdeJVNRg4r3MxhPEK2aw8zoGG1d/YyOjGCVz2XF0rlEvR42bdjP/OUrqYxpOg7tYdehbrTSOK7NjIXvoKG2ED/ew46tzfSNpfF8CMYqWbJ8ISVeL6+/tovpN9/FjAKT1FA7W7fuZSjl4mtNKL+ahUvmUx6D9h2vsCtZSKWK0zcaJ+VY1C5sZMn0smyyePK0aNKjPezZtoOjI2lsO0heWR5JX5+cnxrpZt/OnRwbcVHawyqoYtHiRZT73ezcfYiRoVF2bd9Kdf1C5s8opO/gTnYdPk7aNwCDguo5NC6cRjjexfpXDjD39ruYOv6b0Q7HD25jz5F+PE/jqgBT5y1hYV3J6bEKIYQQQojrznWS1PiMDnSTCFTSePM0omaaI9tfp/lQF1Mbaxlo3cWBHoNFK26jLKLob9tDU2uCMIBO0bZzGwPGFJbfOp2ISnFszzbczARNoXyPvqP7aU8XsPymeURUkrYD+xiMpymJ+DiZDP64biHadXB872SMY+k4kYplLFxcgDN8jM2bm9lTWsayqZr0WIKUG2ZZ402Yox1s3LKT/eXzWb78ZoxUP00bt9HeN5XZhT6u4+BrTXq0j70HupiyeDnTi0OMdbdyqH+QdE0h8bY9dKYLWb5yDlE9xpH9Rxkey1ASPBGnRrvD7GnaQSJ/Disaq7C9UQ5u38bW5jzuvKkW38kwNjBC4fIlNOSbdB/YxrbdLcyqLSHfMk8dJ5rjB7fR4RTRuLKBfMul+/B2DmufWrJNxg7v2kqPUcONN88gRJr2Pa+zvfkwtyybwaIF9cTT7dywZBllhTFMJ05PzyBlsxqZXpGHO9jGhq37OD5tCjOVT8bJnedxuYqX6Wf/3i7KF66gviLCaFcrrcMjOJQQvPwFTgghhBBCXEOukz41iuLahaxcUk/U8kmnPeyQTaJvgKQ3ytEjvZQ2LGR6eRF5eQXUzainvDCWvSeO93K4z6F+wXwqCvPJLyijftZ0ouGJ8z2lTFwnQyqZxLNizF5yE/Vl4TfRxMmgoHgqM6fXkJ+XR0nNbBbUBulu6yIJGIEQVXV1lBXkU1xcSp6tySufRnF+HoUlNVTl24yOJfDGb1IZGNohlUyQcjT5NXN5x6JpREzANPC9JIlECh0ooGHZMqaXRE6L0xk4ztHREPMWzqK4II/8oioWLJhBpvsIfUkNShGtnMG06mLyYgVMqa0lko4z4p/eoV/rIdqPJZk6u4Hq4nzyCoqZ0TCHUjub+LhOHx09DhXlxehMkmTGJ7+inExPO8MpCIUCGKZJKBwmFLBQdgHzV9zG7KoY2sk2gQvqEfqHTzv6M06vQmlNMpEg7WiKaueyfME0Am/4vQghhBBCiMnuOqmpgXhPC8372kn4oLAxM2P4ugTteaQ9j0gkSrYVkkKFQhQEggwBuBnSOkQkbJ6cH4hEiZoTnBrDpLSugTmpPex6fT1eIExxSSVzFsyjJPwGASpFIBwjYOf6eCiTSDSMO5DC1aCUwrJOxHByFRSgc+/PHB0smFfK/BtmsW//Ll7Z5xPLL2bqzHnMmFJIcd0CZg3vYm/TerQdobC0hrnzZ1E8bn3XSYMKEwrlKj2UIhjLJ0KcsSSYgBkMYOeCUoZi4uwtg+uYhELBk029VDCPgnD2vdZpUkMjtOzeTrt1YgM+VkEZlppgxDNvjNbmXbQNjGJoUAGTsbRL/nlOr2mVMn/xdPa07OHVwxAtKKJ65hwaphRLnxohhBBCiOvcdZLUjHJw2x5SlYtZVl9K0LaIt21l8AiAwlSKVCqJ1gGU0uh0mtFMJvucF8PEJE0q7aOjJgqNk0yQ8M6uFdAazGAB9Tcsp3ZWinRylINbN7NjbyG3LM3W1ni+n92u1njjkxCtcVIJHFeDpUD7pJJpDMvCuNi7bmVRMmU2N1VOI5VKMXx8D007tpNXeCuV4XxmLV3BtHSKZGKQ/du2s7WlkHdNO7W6adkonSadBh0ChcZLjpEgTCgIb36kZBOlfNLpDFqHsglYZozRpCYKKAIEC4uYtewmphTlGoNpHw+TaMQmPUw2c9PZl2TPIXYdGWb+yuVUxQLYOs62Vwfe6GRQPHUuN1XVk04lGTi6j21NOymteCflAUlrhBBCCCGuZ9dJ8zPQvk8wFCISCmJkRmg/1kvaBQIxplQX03VgD8eHxsikU3S2H6F3JJ5NPmKlTC1UtOw7wOBYinRimNYjbSQnGl5Ye3Qe2M6WvcchECaWl09ezMJJO/jEyM9zOdbWRv/QMH0dbbR2DuGdyGuUZnSog/ZjfaQyaUZ6j7CvbYzy2qqTAxZcqORQO02vb2MwrYhEY+QV5GM7Dq7n0nmgiS0HOsEOk5eXR17IxMmcnqYEiqqpCg+zf/9R4sk0qbEhDh44jFVSR3n0zScCShVSXWnR3tJCfzxFJjlK+4FD9Of6E1l2CZXFiuOdvXiGTcA2iHcdZv/hHjIaDMPG8hL0jYyQyrgo7eOrIOFomGDAZKS3k57h+HkfNuqNdrJ5wzZ6ExCO5pGfHwXt4p6nxZoQQgghhLg+XCc1NVFqZ9eyaddGVrdHMa0AYWWT7ctuUzF7IXVDG3l97WoiYZtALI/CaH62hsSMMWvxDQxs3MGra44QtExihTFiocDZzZaUQbQgwtDWZtZ1HsDw0oz5eSxdWUOIIHUN9bRv2sna7hZieYWEg0GME+3JlEFefiEjR5tZuz9DOpEiUjmH+bWFKPrPbtalzh6KWI17pwA7lEfM28em19YSthTpsSSlDe+gPC9IOj+PkW3bWNO+H1NnSKsYSxaVA/2nmogFCpm7ZD5N23aw7vheTN/BCxSysHEawdz+zhw5bKIBkhUWtfMW0/v6Ntav6yZoGoSKSqmIZPvUGFaQmfMXMLqlmXVrjhAwPJJpizlL67ANhRkpprbEYvfmDYzMbmRxfS21BRvYsmY14aBJIBjEDoRPZuDjYzh5ekN5hPUQ29avIRgwSSUdyusXURaSWhohhBBCiOudejNPcb/SGhsbdVNT0yVtQ/tutsN8xsUKRgnZCl8rAnY2b/O9DKlEioyvCEfDmNpHGzYBy0BrHzeTJplK4xs20XAQ7XkYdgDrtLZhGq01TiZFMpFGGxbBUIhgwMIAtPZx0imSjiYUCmLi4ymLoG3iOWlcbWLgkBhLgRUkHA5hm9leM07GwbACWGa2aVomk0FZQWxTnXwGj29Y2AY4jodp25gKPM8hlUiQ9sAOZLdpGtltOOkkyVQGrSyC4RBB28LQPhnHPXls2WNPkUym8ZVNKBIiaJmgwHcyuGTjz55jj4zjYQVszDOSHZ17Dk1yLIVnWIQjIQzPA8vGNrPn2HOygxpktEEoFCYUtLNJn9Z4nkM6mYJAlHDAwHfSJJIpXG0QjoQxtI+yAtjKJ+24WHYAQ/lkMi6WHcRQ+tR37PlYgTDhUCD7TB0Z0lkIIYQQYtJrbGykqalpwhu76yapEUIIIYQQQly/zpfUvGGfGqXUQ0qpHqXUrnHTipVSLyilDuZei3LTlVLqG0qpFqXUTqXU0st3GEIIIYQQQghxtjczUMAPgVVnTPsK8JLWehbwUu4zwN3ArNy/+4FvX54whRBCCCGEEGJib5jUaK3XAWeOp3sP8HDu/cPAveOm/0hnbQQKlVJVlylWIYQQQgghhDjLxQ7pXKG17sy97wIqcu9rgPZxyx3LTTuLUup+pVSTUqqpt7f3IsMQQgghhBBCvN1d8nNqdHakgQsebUBr/aDWulFr3VhWVnapYQghhBBCCCHepi42qek+0aws99qTm94BTB233JTcNCGEEEIIIYS4Ii42qXkSuC/3/j7giXHTP5MbBW0FMDyumZoQQgghhBBCXHbWGy2glPoJcDtQqpQ6BvwN8PfAz5VSnwfagI/nFn8aeB/QAiSAz12BmIUQQgghhBDipDdMarTWv32OWXdNsKwG/uRSgxJCCCGEEEKIN+uSBwoQQgghhBBCiKtJkhohhBBCCCHEpCZJjRBCCCGEEGJSk6RGCCGEEEIIMam94UABbwXf9xmNx692GEIIIYQQQohrlO/755x3TSQ1hmEQi0avdhhCCCGEEEKIa5RhnLuR2TWR1AAopa52CEIIIYQQQohJSPrUCCGEEEIIISY1SWqEEEIIIYQQk5okNUIIIYQQQohJTWmtr3YMKKVGgf1XOw4xKZUCfVc7CDEpSdkRF0vKjrhYUnbExZKyk1WntS6baMa1MlDAfq1149UOQkw+SqkmKTviYkjZERdLyo64WFJ2xMWSsvPGpPmZEEIIIYQQYlKTpEYIIYQQQggxqV0rSc2DVzsAMWlJ2REXS8qOuFhSdsTFkrIjLpaUnTdwTQwUIIQQQgghhBAX61qpqRFCCCGEEEKIi3LVkxql1Cql1H6lVItS6itXOx5xbVFKTVVKrVZK7VFK7VZK/WluerFS6gWl1MHca1FuulJKfSNXnnYqpZZe3SMQV5NSylRKbVNK/Sb3ebpSalOufPxMKRXITQ/mPrfk5k+7qoGLq04pVaiU+qVSap9Saq9S6ia57og3opT6s9zfql1KqZ8opUJy3RHnopR6SCnVo5TaNW7aBV9nlFL35ZY/qJS672ocy7XgqiY1SikT+BZwNzAP+G2l1LyrGZO45rjAl7XW84AVwJ/kyshXgJe01rOAl3KfIVuWZuX+3Q98+60PWVxD/hTYO+7zPwBf11rXA4PA53PTPw8M5qZ/PbeceHv7V+BZrXUDsIhsOZLrjjgnpVQN8EWgUWu9ADCBTyLXHXFuPwRWnTHtgq4zSqli4G+AG4HlwN+cSITebq52Tc1yoEVrfVhrnQF+CtxzlWMS1xCtdafWemvu/SjZG4sasuXk4dxiDwP35t7fA/xIZ20ECpVSVW9t1OJaoJSaAvwW8L3cZwXcCfwyt8iZ5eZEefolcFduefE2pJQqAG4Dvg+gtc5orYeQ6454YxYQVkpZQAToRK474hy01uuAgTMmX+h15r3AC1rrAa31IPACZydKbwtXO6mpAdrHfT6WmybEWXJV80uATUCF1rozN6sLqMi9lzIlTvgX4C8BP/e5BBjSWru5z+PLxslyk5s/nFtevD1NB3qBH+SaL35PKRVFrjviPLTWHcA/AUfJJjPDwBbkuiMuzIVeZ+T6k3O1kxoh3hSlVAz4FfAlrfXI+Hk6O4SfDOMnTlJKvR/o0VpvudqxiEnJApYC39ZaLwHGONUEBJDrjjhbrsnPPWST4mogytv0F3Nxech15sJc7aSmA5g67vOU3DQhTlJK2WQTmke11o/lJnefaN6Re+3JTZcyJQBuBj6olDpCtlnrnWT7SBTmmoXA6WXjZLnJzS8A+t/KgMU15RhwTGu9Kff5l2STHLnuiPN5F9Cqte7VWjvAY2SvRXLdERfiQq8zcv3JudpJzWZgVm5kkADZDnVPXuWYxDUk1774+8BerfU/j5v1JHBihI/7gCfGTf9MbpSQFcDwuGpc8Tahtf5rrfUUrfU0steVl7XWnwJWAx/NLXZmuTlRnj6aW15+HXub0lp3Ae1KqTm5SXcBe5Drjji/o8AKpVQk97frRLmR6464EBd6nXkOeI9SqihXW/ie3LS3nav+8E2l1PvItn03gYe01l+9qgGJa4pS6hbgFaCZU30j/l+y/Wp+DtQCbcDHtdYDuT8k/0a2yj8BfE5r3fSWBy6uGUqp24G/0Fq/Xyk1g2zNTTGwDfhdrXVaKRUC/pNsn60B4JNa68NXKWRxDVBKLSY7yEQAOAx8juwPgXLdEeeklPpb4BNkR+7cBvw+2f4Nct0RZ1FK/QS4HSgFusmOYvY4F3idUUr9Htl7I4Cvaq1/8BYexjXjqic1QgghhBBCCHEprnbzMyGEEEIIIYS4JJLUCCGEEEIIISY1SWqEEEIIIYQQk5okNUIIIYQQQohJTZIaIYQQQgghxKQmSY0QQgghhBBiUpOkRgghhBBCCDGpSVIjhBBCCCGEmNT+f4Rlg04GJECoAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1008x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(14,6))\n",
    "x = plt.imread('stacked1/x1.PNG')\n",
    "plt.imshow(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Using XGB with grid search on data with target values less than 200 and selecting top 260 features model has r2 score of 0.55341 which is like top 1% in leaderboard . so using this model as the final model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Final function 1 -- Prediting time taken"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def final1(data):\n",
    "    \"\"\"Given data frame or a single data point it returns\n",
    "        the time taken to complete the testing phase of a \n",
    "        car configuration.\"\"\"\n",
    "    ##loading the top features\n",
    "    best_features = joblib.load('final/top_features.npz')\n",
    "    \n",
    "    ##Loading the model\n",
    "    best_model = joblib.load('final/best_model1.pkl')\n",
    "    \n",
    "    ##for a single data point\n",
    "    if len(data.shape) == 1:\n",
    "        ##Categorical columns\n",
    "        cat_col = []\n",
    "        ##Binary columns\n",
    "        bin_col = []\n",
    "        ##Dataframe to store processed data\n",
    "        t = pd.DataFrame()\n",
    "        \n",
    "        ## Splitting Categorical and binary features\n",
    "        for i in data.index:\n",
    "            if type(data[i]) == str:\n",
    "                cat_col.append(i)\n",
    "            if type(data[i]) == np.int64:\n",
    "                bin_col.append(i)\n",
    "                \n",
    "        ##Encoding categorical features with that particular encoder\n",
    "        for i in cat_col:\n",
    "            en = joblib.load('final1/en_'+str(i)+'.pkl')\n",
    "            t[i]=en.transform(np.array(test[i].iloc[0]).reshape(1,-1))\n",
    "            \n",
    "        ##Merging binary data to the preprocessed data frame\n",
    "        t[data[bin_col].index] = data[bin_col].values\n",
    "        total_data = t\n",
    "        \n",
    "        ##Final data with selected features\n",
    "        final_data = total_data[best_features]\n",
    "        \n",
    "        ##Predicing \n",
    "        pred = best_model.predict(final_data.values[0].reshape(1,-1))\n",
    "        \n",
    "        return pred[0]\n",
    "    \n",
    "    \n",
    "    ##For a set of points or dataframe\n",
    "    if len(data) > 1:\n",
    "        ##Categorical columns\n",
    "        cat_col = []\n",
    "        ##Binary columns\n",
    "        bin_col = []\n",
    "        \n",
    "        ## Splitting Categorical and binary features\n",
    "        for i in data.columns:\n",
    "            if data[i].dtype == 'O':\n",
    "                cat_col.append(i)\n",
    "            if data[i].dtype =='int64':\n",
    "                bin_col.append(i)\n",
    "                \n",
    "        ##Encoding categorical features with its encoder\n",
    "        cat_data=pd.DataFrame()\n",
    "        for i in cat_col:\n",
    "            en = joblib.load('final/en_'+str(i)+'.pkl')\n",
    "            cat_data[i] = en.transform(data[i])\n",
    "        \n",
    "        ##Merging encoded features with binary features\n",
    "        total_data = pd.concat((cat_data,data[bin_col]),axis=1)\n",
    "        \n",
    "        ##Final data with selected features\n",
    "        final_data = total_data[best_features]\n",
    "        \n",
    "        ##Predicting\n",
    "        pred = best_model.predict(final_data.values)\n",
    "        \n",
    "        return pred"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Predicing the time taken on a random data point from test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "86.06186"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final(test.iloc[2])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Predicting the time taken on test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 86.11172 , 102.25736 ,  86.23851 , ...,  92.475975, 110.74211 ,\n",
       "        91.966866], dtype=float32)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final(test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Final funtion 2 -- Predicting metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def final2(data,target):\n",
    "    \"\"\"Given data and target it returns the R2 score \"\"\"\n",
    "    \n",
    "    ##Loading top features\n",
    "    best_features = joblib.load('final/top_features.npz')\n",
    "    \n",
    "    ##Loading the model\n",
    "    best_model = joblib.load('final/best_model1.pkl')\n",
    "    \n",
    "    ##Removing the outliers\n",
    "    outlier_index = joblib.load('final/outlier_index.npz')\n",
    "    data1 = data.iloc[outlier_index]\n",
    "    target1 = target.iloc[outlier_index]\n",
    "    \n",
    "    ##Categorical columns\n",
    "    cat_col = []\n",
    "    \n",
    "    ##Binary features\n",
    "    bin_col = []\n",
    "    \n",
    "    ##Splitting categorical and binary features\n",
    "    for i in data1.columns:\n",
    "        if data1[i].dtype == 'O':\n",
    "            cat_col.append(i)\n",
    "        if data1[i].dtype =='int64':\n",
    "            bin_col.append(i)\n",
    "            \n",
    "    ##Data frame to store processed data\n",
    "    t = pd.DataFrame()\n",
    "    \n",
    "    ##Encoding categorical features with its encoder\n",
    "    for i in cat_col:\n",
    "        en = joblib.load('final1/en_'+str(i)+'.pkl')\n",
    "        t[i] = en.transform(data1[i])\n",
    "    \n",
    "    ##Merging encoded features with binary features\n",
    "    total_data = pd.concat((t,data1[bin_col]),axis=1)\n",
    "    \n",
    "    ##Final data with top selected features\n",
    "    final_data = total_data[best_features]\n",
    "    \n",
    "    ##Prediciting\n",
    "    pred = best_model.predict(final_data.values)\n",
    "    \n",
    "    ##Measuring metric value (r2)\n",
    "    score  = r2_score(target,pred)\n",
    "    \n",
    "    return score\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Given train data and target values predicting the r2 metric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5938214501006536"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final2(train,y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Summary:\n",
    "- Used different combinations of regressors in stacked regressions with XGB as meta regressor for every stack using grid search.\n",
    "- Using best estimators predicted the r2 score for every stack\n",
    "- Instead of using stacked models XGB with grid search on top 260 features with target values less than 200 has best r2 score of 0.55341 which is like top 1% in Private LB\n",
    "- So used this XGB model with best parameters as final model "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "convenv",
   "language": "python",
   "name": "convenv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}