Ensemble.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from scipy import stats\n",
    "from sklearn import metrics\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "import seaborn as sns\n",
    "from misc_functions import *\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.preprocessing import PolynomialFeatures\n",
    "import xgboost as xgb\n",
    "from sklearn.linear_model import Ridge\n",
    "from sklearn.linear_model import Lasso\n",
    "from sklearn.linear_model import ElasticNet\n",
    "from sklearn.linear_model import BayesianRidge\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.linear_model import LassoLarsIC\n",
    "from sklearn.ensemble import GradientBoostingRegressor\n",
    "sns.set_palette('deep')\n",
    "\n",
    "df = pd.read_csv('dataset/train.csv')\n",
    "df = df[df['GrLivArea'] < 4000]\n",
    "y_train = df['SalePrice'].values\n",
    "df_test = pd.read_csv('dataset/test.csv')\n",
    "test_split = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_cum = pd.read_csv('df_cumulative.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((1456, 324), (1459, 324))"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_cum = df_cum.to_numpy()\n",
    "standardScalerX = StandardScaler()\n",
    "X_cum = standardScalerX.fit_transform(X_cum)\n",
    "\n",
    "X_train = X_cum[:1456, :]\n",
    "X_test = X_cum[1456:, :]\n",
    "X_train.shape, X_test.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Feature importance and getting rid of unnecessary ones"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "XGBRegressor(base_score=0.5, booster=None, colsample_bylevel=1,\n",
       "             colsample_bynode=1, colsample_bytree=0.4, gamma=0, gpu_id=-1,\n",
       "             importance_type='gain', interaction_constraints=None,\n",
       "             learning_rate=0.07, max_delta_step=0, max_depth=3,\n",
       "             min_child_weight=1.5, missing=nan, monotone_constraints=None,\n",
       "             n_estimators=400, n_jobs=-1, nthread=-1, num_parallel_tree=1,\n",
       "             objective='reg:linear', random_state=42, reg_alpha=0.75,\n",
       "             reg_lambda=0.45, scale_pos_weight=1, seed=42, silent=True,\n",
       "             subsample=0.6, tree_method=None, validate_parameters=False,\n",
       "             verbosity=None)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xg_reg = xgb.XGBRegressor(base_score=0.5, colsample_bylevel=1, colsample_bytree=0.4,\n",
    "       gamma=0, learning_rate=0.07, max_delta_step=0, max_depth=3,\n",
    "       min_child_weight=1.5, missing=None, n_estimators=400, nthread=-1,\n",
    "       objective='reg:linear', reg_alpha=0.75, reg_lambda=0.45,\n",
    "       scale_pos_weight=1, seed=42, silent=True, subsample=0.6)\n",
    "xg_reg.fit(X_train,np.log(y_train))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_ax = np.arange(len(xg_reg.feature_importances_))\n",
    "# plt.figure(figsize=(90, 30))\n",
    "# sns.barplot(x=x_ax, y=xg_reg.feature_importances_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols = []\n",
    "for i,j in enumerate(xg_reg.feature_importances_):\n",
    "    if j==0:\n",
    "        cols.append(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_cum_n = np.delete(X_cum, cols, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2915, 218)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_cum_n.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = X_cum_n[:1456, :]\n",
    "X_test = X_cum_n[1456:, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "if test_split:\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X_train, y_train, test_size=0.2, random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x124b1c640>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAD4CAYAAAD//dEpAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3de3hc9X3n8fdXd18keUaSbfmCLfku+QYIA4Y4kHAxhFsKDaZpA0latpuy3d1unm1IskmAPG2z291N90lolzTeh+TZJtmQ0JiGlJImJL5wsQ3GxndZvsmSLVuSbfkmS5rv/jFH8iBG0pFlaaTR5/U8ejRz5pyZ7+859nx0zu/8fsfcHRERkb5kpLoAEREZGRQYIiISigJDRERCUWCIiEgoCgwREQklK9UFXAnFxcU+c+bMVJchIjKibN68+YS7l4RdPy0CY+bMmWzatCnVZYiIjChmdrA/6+uUlIiIhKLAEBGRUBQYIiISigJDRERCUWCIiEgoCgwREQlFgSEiIqGM6sDYXtPI8z/fgaZ4FxHp26gOjL2HT/LCr/Zy5nxbqksRERn2QgWGma00s91mVm1mX0jy+goze9vM2s3soW6vPWpme4OfR4Nl+Wa2JeHnhJl9M3jtMTM7nvDaH16JhiZTVJAHQNPpC4P1ESIiaaPPqUHMLBP4NnA7UAtsNLM17r4jYbVDwGPA57ttGwW+ClQBDmwOtm0Gliastxn4acKmP3L3Jy6rRf0QKcgFoPn0BWZMLhjsjxMRGdHCHGEsA6rdvcbdLwI/BO5PXMHdD7j7ViDWbds7gVfdvSkIiVeBlYkrmNkcYCKw9jLbcNmiXUcYrUP90SIiI06YwJgKHE54XhssCyPMto8QP6JI7Hl+0My2mtkLZjY92Rub2eNmtsnMNh0/fjxkOe8XCQKjWaekRET6FCYwLMmysJcVhdl2FfCDhOcvATPdfTHwS+D5ZG/s7s+5e5W7V5WUhJ6d933G5GYxJjeTphYFhohIX8IERi2Q+Ff+NKAu5Pv3uq2ZLQGy3H1z5zJ3b3T3znNE3wGuDflZlyWSn0fTKQWGiEhfwgTGRmCOmZWZWQ7xI4I1Id//FeAOM4uYWQS4I1jW6RHef3SBmZUmPL0P2Bnysy5LpCCP5hb1YYiI9KXPq6Tcvd3MniD+RZ8JrHb37Wb2NLDJ3deY2XXAi0AEuNfMnnL3SndvMrNniIcOwNPu3pTw9p8A7u72kX9qZvcB7UAT8auvBk20II/q2pOD+REiImkh1B333P1l4OVuy76S8Hgj8dNNybZdDazu4bXyJMueBJ4MU9eVECnIVae3iEgIo3qkN8QH71242MG5CxrtLSLSm1EfGF2X1qofQ0SkV6M+MKL5mh5ERCSMUR8YidODiIhIz0Z9YEQ1AaGISCijPjDGjckmOytD80mJiPRh1AeGmcUH7+kIQ0SkV6M+MACi+bk6JSUi0gcFBp3TgygwRER6o8AgPnhPfRgiIr1TYBA/wjh7vo3Wto5UlyIiMmwpMICoxmKIiPRJgUHinfd0WkpEpCcKDDR4T0QkDAUG8bvugQJDRKQ3CgygYFwOmRmmS2tFRHqhwAAyMowJGrwnItIrBUYgPj2IOr1FRHqiwAhE8/N0hCEi0gsFRiBaqOlBRER6o8AIRPNzOXXmIu0dsVSXIiIyLCkwAp2D907q3t4iIkkpMAIavCci0jsFRqDz3t4KDBGR5BQYgWjXfFIKDBGRZBQYgQnjczFD98UQEemBAiOQmZlB4bhcXVorItIDBUaCSIGmBxER6YkCI0G0IE99GCIiPVBgJIjq3t4iIj1SYCSIFORx8kwrHTFPdSkiIsOOAiNBND+XWMw5fUZHGSIi3SkwEkQ02ltEpEcKjARdg/c0n5SIyAcoMBLoCENEpGcKjASR/Ph8Urq0VkTkgxQYCXKyMxk/JltHGCIiSSgwuonfeU99GCIi3YUKDDNbaWa7zazazL6Q5PUVZva2mbWb2UPdXnvUzPYGP48Gy/LNbEvCzwkz+2bwWq6Z/Sj4rDfNbObAmxme7u0tIpJcn4FhZpnAt4G7gArgETOr6LbaIeAx4B+6bRsFvgpcDywDvmpmEXdvcfelnT/AQeCnwWafBZrdfTbwP4FvXG7jLkekIFd9GCIiSYQ5wlgGVLt7jbtfBH4I3J+4grsfcPetQPcbYt8JvOruTe7eDLwKrExcwczmABOBtcGi+4Hng8cvAB81M+tHmwakc3oQd432FhFJFCYwpgKHE57XBsvCCLPtI8CP/NI3dNc27t4OnAKKur+xmT1uZpvMbNPx48dDltO3SEEe7R0xWs61XbH3FBFJB2ECI9lf92H//A6z7SrgB/39PHd/zt2r3L2qpKQkZDl9i+brznsiIsmECYxaYHrC82lAXcj373VbM1sCZLn75mTbmFkWUAg0hfy8AdO9vUVEkgsTGBuBOWZWZmY5xI8I1oR8/1eAO8wsYmYR4I5gWadHeP/RBcF7Pxo8fgj4lQ9hh8Kl6UEUGCIiibL6WsHd283sCeJf9JnAanffbmZPA5vcfY2ZXQe8CESAe83sKXevdPcmM3uGeOgAPO3uiUcLnwDu7vaR3wW+b2bVxI8sVg2ohf10aXoQjcUQEUnUZ2AAuPvLwMvdln0l4fFG4qebkm27Gljdw2vlSZZdAH43TF2DYUxuFmNys9SHISLSjUZ6JxHVvb1FRD5AgZFEpEDTg4iIdKfASCKan0fTKR1hiIgkUmAkESnIo6nlgkZ7i4gkUGAkES3IpfViB+db21NdiojIsKHASEJ33hMR+SAFRhKXpgdRx7eISCcFRhKaHkRE5IMUGElEC8cAmh5ERCSRAiOJcXlZ5GRlaHoQEZEECowkzCx+aa3GYoiIdFFg9CBakKdTUiIiCRQYPYhoPikRkfdRYPQgmp+nGWtFRBIoMHoQKcjj7IV2Wts6Ul2KiMiwoMDoQTQYi6GjDBGROAVGDzQ9iIjI+ykwetB1b2+NxRARARQYPYrqCENE5H0UGD3IH5tDZoYpMEREAgqMHmRkGJF8jcUQEemkwOhFpEBjMUREOikwehGfHkSd3iIioMDoVaQgT6ekREQCCoxeRPNzOX32Im3tsVSXIiKScgqMXnQO3jup01IiIgqM3kQLg8F7muZcRESB0ZtovgbviYh0UmD0IhJMQKjAEBFRYPRqwvhczBQYIiKgwOhVZmYGheNzNQGhiAgKjD5F8zUWQ0QEFBh9ihTk6iopEREUGH2Kaj4pERFAgdGnSEEeJ1ta6Yh5qksREUkpBUYfogV5xBxOn1HHt4iMbgqMPkQ1FkNEBFBg9CmiW7WKiAAKjD5dmh5Ep6REZHQLFRhmttLMdptZtZl9IcnrK8zsbTNrN7OHur32qJntDX4eTVieY2bPmdkeM9tlZg8Gyx8zs+NmtiX4+cOBNnIgOqcH0aW1IjLaZfW1gpllAt8GbgdqgY1mtsbddySsdgh4DPh8t22jwFeBKsCBzcG2zcCXgAZ3n2tmGUA0YdMfufsTl9+sKyc7K5P8sdk6JSUio16fgQEsA6rdvQbAzH4I3A90BYa7Hwhe636noTuBV929KXj9VWAl8APgM8D8YPsYcGIgDRlMure3iEi4U1JTgcMJz2uDZWEk3dbMJgTPnwlOZf3YzCYlrPegmW01sxfMbHqyNzazx81sk5ltOn78eMhyLk80P0/zSYnIqBcmMCzJsrCj2HraNguYBqx392uA14G/Dl5/CZjp7ouBXwLPJ3tjd3/O3avcvaqkpCRkOZcnUpBLk/owRGSUCxMYtUDiX/nTgLqQ79/Tto3AOeDFYPmPgWsA3L3R3Tv/nP8OcG3Izxo08elBWnHXaG8RGb3CBMZGYI6ZlZlZDrAKWBPy/V8B7jCziJlFgDuAVzz+zfsScEuw3kcJ+kTMrDRh+/uAnSE/a9BEC/Jo74jRcq4t1aWIiKRMn53e7t5uZk8Q//LPBFa7+3YzexrY5O5rzOw64kcLEeBeM3vK3SvdvcnMniEeOgBPd3aAA38OfN/MvgkcBz4dLP9TM7sPaAeaiF99lVKJg/cKxuWkuBoRkdQIc5UU7v4y8HK3ZV9JeLyR+OmmZNuuBlYnWX4QWJFk+ZPAk2HqGirRhMCYWVqQ4mpERFJDI71D6Bq8p0trRWQUU2CEcGl6EAWGiIxeCowQ8nKzGJObRXOLxmKIyOilwAgpWpCrIwwRGdUUGCFpehARGe0UGCFFC/J0hCEio5oCI6TS4nE0NJ3j3AUN3hOR0UmBEVJFWRExh90Hm1NdiohISigwQpo/I0KGwfb9jakuRUQkJRQYIY3Ny6ZsaiE79zf1vbKISBpSYPRDZVkRuw4209be/T5RIiLpT4HRDxVlRVxs66DmyMlUlyIiMuQUGP1QURa/7fj2Gp2WEpHRR4HRD5GCPEqLx7FDHd8iMgopMPqpsqyIHfubiMV09z0RGV0UGP1UWR6l5dxFahtaUl2KiMiQUmD0U0VZEQA7dHmtiIwyCox+Ki0ex4T8XA3gE5FRR4HRT2YW78eoUWCIyOiiwLgMFWVRGprPc7z5fKpLEREZMgqMy1BR3tmPoaMMERk9FBiXoay0gDG5merHEJFRRYFxGTIzM5g/I6qJCEVkVFFgXKbK8iIOHj3NmXMXU12KiMiQUGBcpoqyItxh5wEdZYjI0LvQ2s7aLUdoaD43ZJ+ZNWSflGbmzoiQlWlsr2nkuorJqS5HREaBtvYONu9qYO07R3hzx1FaL3bw6Xsq+J1b5wzJ5yswLlNudiazp03QiG8RGVTtHTG27DnO2i1HeOO9es5daCd/bA63XjudDy2dQmV58ZDVosAYgIqyItas3UdrWwe52ZmpLkdE0kRHzHlv3wnWbjnChq11tJxrY1xeFjcuKmXF0mksnlNMVubQ9ygoMAagsryIn75Wzd5DzSycNXQpLyLpJxZzdh5oYt2WI6zbWsfJllbycjK5vrKUFVdP5ep5JWRnpfYPUwXGAMyfGb+h0o79TQoMEek3d2f3oWbWbjnC+nfraDx1gZysDKoqJrFi6TSuXTCRvJzh8zU9fCoZgQrG5XDV5HwN4BOR0NydfbWnWLvlCOvePUJD83myMjO4dv5EHrtnKssqJjE2LzvVZSalwBigyrIifvNOLR0xJzPDUl2OiAxD7s6B+tNBSNRRf+IsmRnG0rkl/N6d87l+YSnjxwzPkEikwBigirIov3j9AAfrT1M+tTDV5YjIMHLo6GnWvVvHunePcPjYGTIMFs8u4cFb53DjolIKxuWkusR+UWAMUOdEhNtrGhUYIkJtQ0s8JLYc4eDRFsxgYXkx99xczvJFU5iQn5vqEi+bAmOAJkbGUhIZw479jdz7ofJUlyMiKVB34gzrtsSPJPbXncYsftn9v/n4IpYvnkK0IC/VJV4RCowroGJmEdv2HcfdMVM/hshocLTxbNfppn21pwBYMDPKH92/kJuWTKGocEyKK7zyFBhXQGV5lN+8U8vRxnOUFo9LdTkiMkgams51hcTewycBmHvVBD57XyXLF09hYmRsiiscXAqMK6Ci7NINlRQYIumlofkc64OQ2HMoHhKzp0/g0/dUsHzxFCYXjZ7/86ECw8xWAn8DZAJ/7+5/1e31FcA3gcXAKnd/IeG1R4EvB0+/7u7PB8tzgG8BtwAx4Evu/hMzywW+B1wLNAIPu/uBy23gUJg+KZ/xY7LZXtPIR6+7KtXliMgAHW8+z/qt8ZDYfbAZgFnTCnn0YxXcvGR0hUSiPgPDzDKBbwO3A7XARjNb4+47ElY7BDwGfL7btlHgq0AV4MDmYNtm4EtAg7vPNbMMIBps9lmg2d1nm9kq4BvAwwNo46DLyDAqyoo0EaHICHbi5Hk2bK1j3bt1XbctKJ9ayKfuXsDNS6bq7AHhjjCWAdXuXgNgZj8E7ge6AqPzCMDMYt22vRN41d2bgtdfBVYCPwA+A8wPto8BJ4Jt7ge+Fjx+AfiWmZm7ez/bNqQqyqK8teMoJ1taR/RlcyKjSbKQmFlawB/ctYCbl0xhSsn4FFc4vIQJjKnA4YTntcD1Id8/2bZTzWxC8PwZM7sF2Ac84e7HErdx93YzOwUUcSlQADCzx4HHAa66KvWngSrLL/VjLF88JcXViEhPGk+dD/ok3h8Sv3/XfG5eMpWpCokehQmMZNeJhv1rv6dts4BpwHp3/zMz+zPgr4E/CPt57v4c8BxAVVVVyo8+Zk2bQE5WBjv2NykwRIaZHkNi5XxuWjKFaRPzU1zhyBAmMGqB6QnPpwF1Id+/lninduK2rxHvzD4HvBgs/zHxvovEz6s1syygEBj2nQPZWRnMnRHRRIQiw8SJk/GO6/UKiSsmTGBsBOaYWRlwBFgF/F7I938F+AsziwTP7wCedHc3s5eIh8mvgI9yqU9kDfAo8DrwEPCr4d5/0amyrIgf/2ov51vbGZOrK5ZFhlrn1U3r3z3CruDqppmlBXxy5XxuVkgMWJ/fakE/whPEv/wzgdXuvt3MngY2ufsaM7uO+NFCBLjXzJ5y90p3bzKzZ4iHDsDTnR3gwJ8D3zezbwLHgU8Hy78bLK8mfmSx6gq1ddBVlBURi+1h98Emls6dmOpyREaFhuZzXR3XnZfAlk1Rn8RgCPVnsLu/DLzcbdlXEh5vJH66Kdm2q4HVSZYfBFYkWX4B+N0wdQ0382dGyDDYXqPAEBlMx5rig+nWb700mK7zEtibFuvqpsGi8yZX0Ni8bMqmFrJD/RgiV9zRxrPxjuutdVQH03LMmhaExJIpTClWSAw2BcYVVlFWxL+8eZD2jlhKbtIukk7qjp+J90lsreua4G/O9Ak89rEKbhrFI65TRYFxhVWWFfHS2hpqjpxi7lWRvjcQkfepbWgJTjfVsb/uNADzZkT4zL3xCf4mRdN7gr/hTIFxhVWUxWc42V7TqMAQCcHdOXSshQ1BSBw82gLEpwr/w/sXcuOi0rSfBXakUGBcYZGCPEqLx/HevkY+fsvsVJcjMix13uO680iituEMZsH9JB5YyE2L0/N+EiOdAmMQXF85mTVrazjaeFbnWEUC7s6+2lNdfRL1J86SYbBwVjH3fqicGxeWEkmTO9OlKwXGIPj4LbP5+fr9/Phf9/LvPrE01eWIpEws5uw53MyGrfWs31pHQ9M5MjKMJbOLefDW2dywsJTC8Zqsc6RQYAyCaEEed14/g1+8foCHb5vLRHXSySjSEXN27m9kw7Z6Nmyto/HUBbIyjaVzJ/LI7XNZVllKwbicVJcpl0GBMUh+59Y5/PMbB3jh13v53INLUl2OyKBq74ixrfoEG7bV88a2ek6eaSU7K4Nr5k3kU3dPYVnlZMaPyU51mTJACoxBUhIZw23LZvDqm4d4+La56sCTtNPW3sGWPcfZsLWeN7fX03KujdycTKoWTOKmxVOoWjBJc6qlGe3NQfTQR+bw6psH+cmvq3n8gUWpLkdkwC5cbOed3Q1s2FrPWzuOcu5CO2PzslhWMZnli0u5et5E8nL0tZKutGcH0aToWD5SNZ1XXj/A735kjq4AkRHp3IU2Nu44xoZtdWze1UDrxQ7yx2azfNEUbloyhSVzisnOykx1mTIEFBiD7KGPzuFfNx7ip69V89n7Fqa6HJFQTp+9yFvb69mwrZ53dh+nvSNGJD+Xj1RNZ/miUhbOKtbUN6OQAmOQTSkez4prpvGL1w/w0Efm6BJCGbaaT1/gjffq2bC1nq37ThCLOSWRMXzspjJuXFTK/JlRMjOS3RBTRgsFxhD4xEfn8pu3a/nH3+zj0Y9VpLockS7Hms7x+rZ6Xt8WvyudO0wtGceDt85m+aIpzJpWiJlCQuIUGENg+qR8bl4ylZ+vr+F3bp1N/lhdgy6pc/hYCxu21fH6tvquGWDLphTwyB3zWb6olKsm5yskJCkFxhD5xG1zWbvlCGt+W8MnV85PdTkyirg7+46c4vVgIF1twxkA5gczwN64qFRT2EgoCowhMrO0gBsXlfLS2n088OFZjNMgJhlEHTFn14EmNmyr441t9TQ0nycjw1g0q4h7bi7nhoWTNTZI+k2BMYQevm0ur2+r55/W1fDw7fNSXY6kmbb2Dt7de4LXt8UH0p06c5HsrAyWzi3hkTvms6xysqbkkAFRYAyhWdMmsKxiMj/77T7u/VA5Y/N0lCEDc+5CG5t3NfDGtno27jzG+dZ2xuRmcV3FJG5cVMo18ybq35lcMQqMIfbw7XP5T3/zW17eEL/MVqS/Tp1p5a3tR9mwrZ539x6nrT1G4fgcVlw9lRsWlmognQwaBcYQm3tVhGvmTeQff1PNPTeVkae5diSEo41neeO9o7zxXj079zcSc5gYGcPdyzVGQoaOvq1SYNXt8/jP31rLP79xgAc+rLvyyQe5OzVHTnWFxIH6+L2tZ5YW8Inb5nHDwsmUT9UYCRlaCowUWFAWZfHsYn7662ruWl5GbrZOHwh0dMTYsb+JN96r5433giubDBaUFfHZ+xZyw8LJuvxVUkqBkSKr7pjHF59dz7+8cZB7P1Se6nIkRc63xmd/fXP7UTbuOEbLufiVTVfPnciq2+dxXcVkJuRrOhkZHhQYKbJoVjGV5UX85Nd7WXnjDHVSjiLNpy/w5vajvLn9aFen9fgx2VRVTOKGhfErm3QfCRmO9K8yhR6+bS5fee51XnnjIPfcrKOMdOXuHD7WEg+J946y+1AzABOjY7nrxplcv3AyFWVFmv1Vhj0FRgotnVvC4tnFfHfNe0wuGkfVgkmpLkmukI6OGDsONPFWcCRRf+IsALOnFfLJlfO5vnIyM0sL1GktI4q5e6prGLCqqirftGlTqsu4LGfOt/Hlv1vPoaMt/JfPXM/V8yamuiS5TGfOt/H2rmO8tf0Ym3cd48z5NrIyjcWzS7h+4WSWVUymeIKm45Dhw8w2u3tV6PUVGKl3+uxFvvS366k7cZav/dENLJpVnOqSJKS6E2d4a/sxNu44yvaaRjpiTsG4HKoWTGJZxWSunleikdYybCkwRqiTLa188W/Xc7z5HE89fiMVZUWpLkmS6OiIsetgMxt3xE81dc78etXkfJZVTOa6iknMm6FBdDIyKDBGsKbTF3jy2+tobmnl63+8nLlXRVJdkhCfimPzrgY27zzG27sbuk41LSwv5rrK+JGExkfISKTAGOFOnDzPk8+uo+VcG1//4+XMnjYh1SWNOrFYfJT1xp3H2LzzGHsON+MOE/JzuXb+RKoWTNKkfpIWFBhpoKHpHF94dh0XWtv5i8/dzMzSglSXlPbOnm9jy57jbNoZ77BubmnFDOZOj3DtgklULZjIrKkTyNCpJkkjCow0UX/iLE8+u472jhh/+bmbmT4pP9UlpZWOmLOv9iTv7G7g7d0N7DrYTCzmjBuTzTXzLh1FaJS1pDMFRhqpbWjhyWfXk2Hwl5+7mSkl41Nd0ojWeOo8b+9q4J09x9myp4GWc21AfGzE1fMmcu38ScyfESFTA+hklFBgpJmDR0/zxWfXk5OVwV/+yc3qXO2H1rYOtu9r5J098aOIQ0dbAIjk53L1vInxn7klFI7XUYSMTgqMNLS/7hRffHY9Y/OyeOrxG5k2Uaenkmlr72DPoZNs3XucrftOsPtgM23tMbKzMqgsKwpCokQjrEUCCow0VX34JF/+3xs439rOLddM4xO3zWXqKD9F1d4Ro/rwSbZWn2Bb9Ql2HGjiYlsHZlA+tZBFs4pZMqeEhbOKyMvRLDgi3Q1KYJjZSuBvgEzg7939r7q9vgL4JrAYWOXuLyS89ijw5eDp1939+WD5a0ApcD547Q53bzCzx4D/BhwJln/L3f++t/pGQ2BAfJbTn75WzcsbDtDe3sGKq+PBMVo6xNvaY9QcOcn2mka2Vp9gx/5Gzrd2APEbCy2aXcyiWcUsnFVE/ticFFcrMvxd8cAws0xgD3A7UAtsBB5x9x0J68wECoDPA2s6A8PMosAmoApwYDNwrbs3B4HxeXd/3zd9EBhV7v5E2EaMlsDo1NxygRdf28fLG/Zzsa2DDy2dysO3zeWqyel1+W3T6QvsOtDEroPN7DrQRHXtSdraYwBMmzieRbOLWTI7fgShfgiR/utvYIQ5Tl8GVLt7TfABPwTuB7oCw90PBK/Fum17J/CquzcFr78KrAR+ELZA+aBIfh6fubeSB2+dzYuvVfPz9ftZu+UINy2ewsO3zxuR4zbaO2LsrzvFrgPN7DrYxK4DTTQ0xw8+szIzmD2tkI/dVMb8GVEWlEWJFuSluGKR0SdMYEwFDic8rwWuD/n+ybadmvD8/5hZB/AT4qerOg93HgxOc+0B/qO7J74HAGb2OPA4wFVXXRWynPRSOD6Xx+6p5OO3zOZnv93HP63bz7p361i+uJRVt8+jbEphqktM6vTZixysP82B+tMcPBr/vb/uNBfb4qeXigrzmD8jyr0fKmf+jCizphXqBlMiw0CYwEh2OUnYnvLetv2kux8xs3zigfEHwPeAl4AfuHurmf0x8DzwkQ+8iftzwHMQPyUVsp60VDg+l0/dXREPjt/s46V1NWzYWs/E6FhmTS2kPPiZNbWQaEHekF0hdLGtg0PHWi6FQxAQTadbu9bJH5vNjNICVt4wg/kzosyfGaUkoinARYajMIFRC0xPeD4NqAv5/rXALd22fQ3A3Y8Ev1vM7B+In/r6nrs3Jqz/HeAbIT9r1Msfm8Pv37WABz48i19uPMTug83UHDnF69vqu9YpHJ9D+ZTOAJlA+bRCSovG9WvKC3fn7IV2mk6dp/HUBZpOX0j4fZ6m0xdoCp7HgijPzspg+qR8ls6dyIzJBcwsLWBGaf6QBpiIDEyYwNgIzDGzMuJXLq0Cfi/k+78C/IWZdU67egfwpJllARPc/YSZZQP3AL8EMLNSd+/8hrsP2BnysyQwfmwOD3x4dtfzcxfa2F93mpojp7p+fvbbfbR3xL/Nc7IyyM7OJMOMjAzIMMPMyMgwMoz3Pe6IOc0trbRe7PjA544bk020II+iwjymTcynJDImHgyTC5hSPE4jqEVGuD4Dw93bzewJ4l/+mcBqd99uZk8Dm9x9jZldB7wIRIB7zewpd6909yYze4Z46AA8HSwbB7wShEUm8bD4TrDOn5rZfUA70AQ8duWaO0F3xpEAAAUFSURBVDqNzcumsryIyvJL99hoa+/g0NEWao6c4tCxFto7YsRijjvE3InFnJgHz4PHsZiTYUakIK8rGKKFwe/8PPJyNdZBJJ1p4J6IyCjV38tqdY5ARERCUWCIiEgoCgwREQlFgSEiIqEoMEREJBQFhoiIhKLAEBGRUBQYIiISSloM3DOz48DBhEXFwIkUlTPY0rVtatfIk65tS9d2wQfbNsPdS8JunBaB0Z2ZberP6MWRJF3bpnaNPOnatnRtFwy8bTolJSIioSgwREQklHQNjOdSXcAgSte2qV0jT7q2LV3bBQNsW1r2YYiIyJWXrkcYIiJyhSkwREQklLQLDDNbaWa7zazazL6Q6noGwswOmNk2M9tiZpuCZVEze9XM9ga/I329z3BgZqvNrMHM3ktYlrQtFve/gn241cyuSV3lveuhXV8zsyPBfttiZncnvPZk0K7dZnZnaqrum5lNN7Nfm9lOM9tuZv8+WD6i91kv7UqHfZZnZm+Z2btB254KlpeZ2ZvBPvuRmeUEy3OD59XB6zP7/BB3T5sf4rd73QeUAznAu0BFqusaQHsOAMXdlv1X4AvB4y8A30h1nSHbsgK4Bnivr7YAdwO/AAy4AXgz1fX3s11fAz6fZN2K4N9kLlAW/FvNTHUbemhXKXBN8Dgf2BPUP6L3WS/tSod9ZsD44HE28GawL/4fsCpY/nfAvw0efw74u+DxKuBHfX1Guh1hLAOq3b3G3S8CPwTuT3FNV9r9wPPB4+eBB1JYS2ju/lvi92hP1FNb7ge+53FvABPMrHRoKu2fHtrVk/uBH7p7q7vvB6qJ/5sddty93t3fDh63ADuBqYzwfdZLu3oykvaZu/uZ4Gl28OPAR4AXguXd91nnvnwB+KiZWW+fkW6BMRU4nPC8lt7/MQx3DvyLmW02s8eDZZPcvR7i//iBiSmrbuB6aks67McnglMzqxNOG47IdgWnKq4m/hdr2uyzbu2CNNhnZpZpZluABuBV4kdEJ929PVglsf6utgWvnwKKenv/dAuMZOk4kq8bvsndrwHuAv7EzFakuqAhMtL3498Cs4ClQD3w34PlI65dZjYe+AnwH9z9dG+rJlk2bNuWpF1psc/cvcPdlwLTiB8JLUi2WvC7321Lt8CoBaYnPJ8G1KWolgFz97rgdwPwIvF/AMc6D/WD3w2pq3DAemrLiN6P7n4s+I8bA77DpVMYI6pdZpZN/Ev1/7r7T4PFI36fJWtXuuyzTu5+EniNeB/GBDPLCl5KrL+rbcHrhfRxejXdAmMjMCe4KiCHeEfOmhTXdFnMbJyZ5Xc+Bu4A3iPenkeD1R4FfpaaCq+IntqyBvhUcOXNDcCpztMgI0G3c/cfJ77fIN6uVcHVKWXAHOCtoa4vjOBc9neBne7+PxJeGtH7rKd2pck+KzGzCcHjMcBtxPtofg08FKzWfZ917suHgF950APeo1T37A/ClQJ3E7/yYR/wpVTXM4B2lBO/OuNdYHtnW4ifY/xXYG/wO5rqWkO25wfED/XbiP9l89me2kL8UPnbwT7cBlSluv5+tuv7Qd1bg/+UpQnrfylo127grlTX30u7biZ+emIrsCX4uXuk77Ne2pUO+2wx8E7QhveArwTLy4mHXDXwYyA3WJ4XPK8OXi/v6zM0NYiIiISSbqekRERkkCgwREQkFAWGiIiEosAQEZFQFBgiIhKKAkNEREJRYIiISCj/Hy3QM+GJXqMVAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "                 ###################### Parameter sweeping ##########################\n",
    "val_list = []\n",
    "score_list = []\n",
    "for x in np.arange(10,300,10):\n",
    "    model = Ridge(alpha=x)\n",
    "    model.fit(X_train,np.log(y_train))\n",
    "    y_pred = np.exp(model.predict(X_test))\n",
    "    score_list.append(np.sqrt(metrics.mean_squared_error(np.log(y_test), np.log(y_pred))))\n",
    "    val_list. append(x)\n",
    "    \n",
    "sns.lineplot(x=val_list, y=score_list, markers=True)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_pred = np.array([])\n",
    "n = 8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "                             #### Gradient Boosting Regressor (0.1134)####\n",
    "gbregressor = GradientBoostingRegressor(learning_rate=0.1, n_estimators=180)\n",
    "gbregressor.fit(X_train, np.log(y_train))\n",
    "y_pred = np.exp(gbregressor.predict(X_test))\n",
    "all_pred = np.concatenate((all_pred, 0*y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "                             #### Lasso LarsIC (0.119)####\n",
    "lassolars = LassoLarsIC(criterion='aic')\n",
    "lassolars.fit(X_train, np.log(y_train))\n",
    "y_pred = np.exp(lassolars.predict(X_test))\n",
    "all_pred = np.vstack((all_pred, 0*y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "                             #### Random Forest (0.12401)####\n",
    "rf_regressor = RandomForestRegressor(n_estimators = 325, random_state = 0)\n",
    "rf_regressor.fit(X_train, np.log(y_train))\n",
    "y_pred = np.exp(rf_regressor.predict(X_test))\n",
    "all_pred = np.vstack((all_pred, 0*y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "                             #### Bayesian Ridge (0.1054)####\n",
    "bayesianridge = BayesianRidge(alpha_1=130, alpha_2=0.0017, lambda_1=0.00001, lambda_2=0.000001)\n",
    "bayesianridge.fit(X_train, np.log(y_train))\n",
    "y_pred = np.exp(bayesianridge.predict(X_test))\n",
    "all_pred = np.vstack((all_pred, 0.17*y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "                             #### ElasticNET (0.10032)####\n",
    "elasticnet = ElasticNet(alpha=0.011, l1_ratio=0.5)\n",
    "elasticnet.fit(X_train, np.log(y_train))\n",
    "y_pred = np.exp(elasticnet.predict(X_test))\n",
    "all_pred = np.vstack((all_pred, 0.21*y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "                             #### Lasso (0.10026)####\n",
    "lasso = Lasso(alpha=0.005)\n",
    "lasso.fit(X_train, np.log(y_train))\n",
    "y_pred = np.exp(lasso.predict(X_test))\n",
    "all_pred = np.vstack((all_pred, 0.29*y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "                             #### Ridge Regressor (0.1054)####\n",
    "ridge = Ridge(alpha=75)\n",
    "ridge.fit(X_train, np.log(y_train))\n",
    "y_pred = np.exp(ridge.predict(X_test))\n",
    "all_pred = np.vstack((all_pred, 0.14*y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "                             #### XGBoost Regressor (0.10624)#####\n",
    "xg_reg = xgb.XGBRegressor(base_score=0.5, colsample_bylevel=1, colsample_bytree=0.4,\n",
    "       gamma=0, learning_rate=0.07, max_delta_step=0, max_depth=3,\n",
    "       min_child_weight=1.5, missing=None, n_estimators=400, nthread=-1,\n",
    "       objective='reg:linear', reg_alpha=0.75, reg_lambda=0.45,\n",
    "       scale_pos_weight=1, seed=42, silent=True, subsample=0.6)\n",
    "xg_reg.fit(X_train,np.log(y_train))\n",
    "y_pred = np.exp(xg_reg.predict(X_test))\n",
    "all_pred = np.vstack((all_pred, 0.19*y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = np.sum(all_pred, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_pred = pd.DataFrame({'Actual': y_test.flatten(), 'Predicted': y_pred.flatten()})\n",
    "# df_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "if test_split:\n",
    "    print('Root Mean Squared Error:', np.sqrt(metrics.mean_squared_error(np.log(y_test), np.log(y_pred))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not test_split:\n",
    "    id_array_test = df_test['Id'].values\n",
    "    final_result = np.concatenate([id_array_test.reshape(-1,1), y_pred.reshape(-1,1)], axis=1)\n",
    "\n",
    "    index_array = np.array([x for x in range(1459)])\n",
    "    submission_df = pd.DataFrame(data=final_result, index=index_array, columns=['Id', 'SalePrice'])\n",
    "    convert_dict = {'Id': int}\n",
    "\n",
    "    submission_df = submission_df.astype(convert_dict)\n",
    "\n",
    "    submission_df.to_csv('submission_get_dummies.csv',index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}