prepare-reg-dataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "576051b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import StratifiedKFold, KFold\n",
    "import pickle\n",
    "\n",
    "import os\n",
    "import pandas as pds\n",
    "\n",
    "dataname = 'house-prices'\n",
    "df = pds.read_csv(os.path.join('data',dataname,'train.csv'))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "0afb4136",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.dropna(1).iloc[:,1:]\n",
    "\n",
    "obj2int_dict = dict()\n",
    "for key,col in df.dtypes.iteritems():\n",
    "    if col == object: \n",
    "        obj2int_dict[key] = {v:n for n,v in enumerate(df[key].unique())}\n",
    "        df[key] = df[key].apply(lambda x: obj2int_dict[key][x])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "a7b2bf7d",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MSSubClass</th>\n",
       "      <th>MSZoning</th>\n",
       "      <th>LotArea</th>\n",
       "      <th>Street</th>\n",
       "      <th>LotShape</th>\n",
       "      <th>LandContour</th>\n",
       "      <th>Utilities</th>\n",
       "      <th>LotConfig</th>\n",
       "      <th>LandSlope</th>\n",
       "      <th>Neighborhood</th>\n",
       "      <th>...</th>\n",
       "      <th>OpenPorchSF</th>\n",
       "      <th>EnclosedPorch</th>\n",
       "      <th>3SsnPorch</th>\n",
       "      <th>ScreenPorch</th>\n",
       "      <th>PoolArea</th>\n",
       "      <th>MiscVal</th>\n",
       "      <th>MoSold</th>\n",
       "      <th>YrSold</th>\n",
       "      <th>SaleType</th>\n",
       "      <th>SaleCondition</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1460.000000</td>\n",
       "      <td>1460.000000</td>\n",
       "      <td>1460.000000</td>\n",
       "      <td>1460.000000</td>\n",
       "      <td>1460.000000</td>\n",
       "      <td>1460.000000</td>\n",
       "      <td>1460.000000</td>\n",
       "      <td>1460.000000</td>\n",
       "      <td>1460.000000</td>\n",
       "      <td>1460.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>1460.000000</td>\n",
       "      <td>1460.000000</td>\n",
       "      <td>1460.000000</td>\n",
       "      <td>1460.000000</td>\n",
       "      <td>1460.000000</td>\n",
       "      <td>1460.000000</td>\n",
       "      <td>1460.000000</td>\n",
       "      <td>1460.000000</td>\n",
       "      <td>1460.000000</td>\n",
       "      <td>1460.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>56.897260</td>\n",
       "      <td>0.340411</td>\n",
       "      <td>10516.828082</td>\n",
       "      <td>0.004110</td>\n",
       "      <td>0.408219</td>\n",
       "      <td>0.195205</td>\n",
       "      <td>0.000685</td>\n",
       "      <td>0.596575</td>\n",
       "      <td>0.062329</td>\n",
       "      <td>9.478082</td>\n",
       "      <td>...</td>\n",
       "      <td>46.660274</td>\n",
       "      <td>21.954110</td>\n",
       "      <td>3.409589</td>\n",
       "      <td>15.060959</td>\n",
       "      <td>2.758904</td>\n",
       "      <td>43.489041</td>\n",
       "      <td>6.321918</td>\n",
       "      <td>2007.815753</td>\n",
       "      <td>0.234932</td>\n",
       "      <td>0.35000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>42.300571</td>\n",
       "      <td>0.798309</td>\n",
       "      <td>9981.264932</td>\n",
       "      <td>0.063996</td>\n",
       "      <td>0.582296</td>\n",
       "      <td>0.642010</td>\n",
       "      <td>0.026171</td>\n",
       "      <td>1.010359</td>\n",
       "      <td>0.276232</td>\n",
       "      <td>5.837783</td>\n",
       "      <td>...</td>\n",
       "      <td>66.256028</td>\n",
       "      <td>61.119149</td>\n",
       "      <td>29.317331</td>\n",
       "      <td>55.757415</td>\n",
       "      <td>40.177307</td>\n",
       "      <td>496.123024</td>\n",
       "      <td>2.703626</td>\n",
       "      <td>1.328095</td>\n",
       "      <td>0.804553</td>\n",
       "      <td>0.88787</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>20.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1300.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2006.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>20.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7553.500000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>2007.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>50.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>9478.500000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>25.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>2008.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>70.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>11601.500000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>13.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>68.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>2009.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>190.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>215245.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>24.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>547.000000</td>\n",
       "      <td>552.000000</td>\n",
       "      <td>508.000000</td>\n",
       "      <td>480.000000</td>\n",
       "      <td>738.000000</td>\n",
       "      <td>15500.000000</td>\n",
       "      <td>12.000000</td>\n",
       "      <td>2010.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>5.00000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 60 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        MSSubClass     MSZoning        LotArea       Street     LotShape  \\\n",
       "count  1460.000000  1460.000000    1460.000000  1460.000000  1460.000000   \n",
       "mean     56.897260     0.340411   10516.828082     0.004110     0.408219   \n",
       "std      42.300571     0.798309    9981.264932     0.063996     0.582296   \n",
       "min      20.000000     0.000000    1300.000000     0.000000     0.000000   \n",
       "25%      20.000000     0.000000    7553.500000     0.000000     0.000000   \n",
       "50%      50.000000     0.000000    9478.500000     0.000000     0.000000   \n",
       "75%      70.000000     0.000000   11601.500000     0.000000     1.000000   \n",
       "max     190.000000     4.000000  215245.000000     1.000000     3.000000   \n",
       "\n",
       "       LandContour    Utilities    LotConfig    LandSlope  Neighborhood  ...  \\\n",
       "count  1460.000000  1460.000000  1460.000000  1460.000000   1460.000000  ...   \n",
       "mean      0.195205     0.000685     0.596575     0.062329      9.478082  ...   \n",
       "std       0.642010     0.026171     1.010359     0.276232      5.837783  ...   \n",
       "min       0.000000     0.000000     0.000000     0.000000      0.000000  ...   \n",
       "25%       0.000000     0.000000     0.000000     0.000000      5.000000  ...   \n",
       "50%       0.000000     0.000000     0.000000     0.000000     10.000000  ...   \n",
       "75%       0.000000     0.000000     1.000000     0.000000     13.000000  ...   \n",
       "max       3.000000     1.000000     4.000000     2.000000     24.000000  ...   \n",
       "\n",
       "       OpenPorchSF  EnclosedPorch    3SsnPorch  ScreenPorch     PoolArea  \\\n",
       "count  1460.000000    1460.000000  1460.000000  1460.000000  1460.000000   \n",
       "mean     46.660274      21.954110     3.409589    15.060959     2.758904   \n",
       "std      66.256028      61.119149    29.317331    55.757415    40.177307   \n",
       "min       0.000000       0.000000     0.000000     0.000000     0.000000   \n",
       "25%       0.000000       0.000000     0.000000     0.000000     0.000000   \n",
       "50%      25.000000       0.000000     0.000000     0.000000     0.000000   \n",
       "75%      68.000000       0.000000     0.000000     0.000000     0.000000   \n",
       "max     547.000000     552.000000   508.000000   480.000000   738.000000   \n",
       "\n",
       "            MiscVal       MoSold       YrSold     SaleType  SaleCondition  \n",
       "count   1460.000000  1460.000000  1460.000000  1460.000000     1460.00000  \n",
       "mean      43.489041     6.321918  2007.815753     0.234932        0.35000  \n",
       "std      496.123024     2.703626     1.328095     0.804553        0.88787  \n",
       "min        0.000000     1.000000  2006.000000     0.000000        0.00000  \n",
       "25%        0.000000     5.000000  2007.000000     0.000000        0.00000  \n",
       "50%        0.000000     6.000000  2008.000000     0.000000        0.00000  \n",
       "75%        0.000000     8.000000  2009.000000     0.000000        0.00000  \n",
       "max    15500.000000    12.000000  2010.000000     8.000000        5.00000  \n",
       "\n",
       "[8 rows x 60 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = df.drop(columns = 'SalePrice')\n",
    "y = df.SalePrice\n",
    "\n",
    "X.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "bb00f20b",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "ss1 = KFold(6, shuffle = True, random_state = 523)\n",
    "ss2 = KFold(5, shuffle = True, random_state = 523)\n",
    "\n",
    "for train_idx, test_idx in ss1.split(X,y):\n",
    "    train_X, train_y = X.iloc[train_idx], y.iloc[train_idx]\n",
    "    test_X, test_y = X.iloc[test_idx], y.iloc[test_idx]\n",
    "    \n",
    "    \n",
    "    for train_idx, valid_idx in ss2.split(train_X,train_y):\n",
    "        valid_X, valid_y = train_X.iloc[valid_idx], train_y.iloc[valid_idx]\n",
    "        train_X, train_y = train_X.iloc[train_idx], train_y.iloc[train_idx]\n",
    "        break\n",
    "    break\n",
    "\n",
    "with open(os.path.join('data',f'{dataname}.pkl'),'wb') as f:\n",
    "    pickle.dump(\n",
    "        dict(\n",
    "            train_x = train_X.values,\n",
    "            train_y = train_y.values,\n",
    "            valid_x = valid_X.values,\n",
    "            valid_y = valid_y.values,\n",
    "            test_X = test_X.values,\n",
    "            test_y = test_y.values,\n",
    "            columns = train_X.columns,\n",
    "        ),f\n",
    "    )"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}