FeatureEngineering.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Config finished!\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "#setup\n",
    "import os, sys\n",
    "current_path = os.path.abspath('')\n",
    "# parent_path = os.path.dirname(current_path)\n",
    "parent_path = current_path\n",
    "source_path = f'{parent_path}\\\\src'\n",
    "if not source_path in sys.path: sys.path.append(source_path)\n",
    "\n",
    "#customs\n",
    "# from data_model import *\n",
    "# from data_helper import *\n",
    "from config import *\n",
    "setup()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Value</th>\n",
       "      <th>League</th>\n",
       "      <th>Confidence</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Date</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2016-09-08</th>\n",
       "      <td>64.00417</td>\n",
       "      <td>Essence</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2016-09-09</th>\n",
       "      <td>64.51451</td>\n",
       "      <td>Essence</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2016-09-10</th>\n",
       "      <td>65.00280</td>\n",
       "      <td>Essence</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2016-09-11</th>\n",
       "      <td>65.05764</td>\n",
       "      <td>Essence</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2016-09-12</th>\n",
       "      <td>65.98811</td>\n",
       "      <td>Essence</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               Value   League  Confidence\n",
       "Date                                     \n",
       "2016-09-08  64.00417  Essence           0\n",
       "2016-09-09  64.51451  Essence           0\n",
       "2016-09-10  65.00280  Essence           0\n",
       "2016-09-11  65.05764  Essence           0\n",
       "2016-09-12  65.98811  Essence           0"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('./data-training/Exalted Orb.csv', index_col='Date', parse_dates=True)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Engineer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_lags(ts, lags, lead_time=1):\n",
    "    return pd.concat(\n",
    "        {\n",
    "            f'y_lag_{i}': ts.shift(i)\n",
    "            for i in range(lead_time, lags + lead_time)\n",
    "        },\n",
    "        axis=1)\n",
    "def make_multi_target(ts, steps, lead_time=1):\n",
    "    return pd.concat(\n",
    "        {\n",
    "            f'y_step_{i}': ts.shift(-i)\n",
    "            for i in range(lead_time, steps + lead_time)\n",
    "        },\n",
    "        axis=1)\n",
    "\n",
    "class FeatureEngineer:\n",
    "    def target(self, df, steps = 5):\n",
    "        return pd.concat([\n",
    "            make_multi_target(df['Value'], steps).fillna(0),\n",
    "            df[['League']]\n",
    "        ], axis = 1)\n",
    "    \n",
    "    def time_features(self, df):\n",
    "        df_return = pd.DataFrame()\n",
    "        df_return['LeagueDay'] = (df.groupby('League').cumcount() + 1).astype(np.float64)\n",
    "        time_index = df.index\n",
    "\n",
    "\n",
    "        df_return['day_of_week'] = time_index.dayofweek\n",
    "        df_return['month'] = time_index.month\n",
    "\n",
    "        return df_return\n",
    "\n",
    "    def lag_features(self, df, lags = 4):\n",
    "        return make_lags(df['Value'], lags).fillna(0)\n",
    "\n",
    "    def rolling_features(self, df, local_window_size = 5, general_window_size=15):\n",
    "        local_rolling = df['Value'].rolling(local_window_size, center = True)\n",
    "        general_rolling = df['Value'].rolling(general_window_size, center = True)\n",
    "\n",
    "        df_return = pd.DataFrame()\n",
    "        df_return['rolling_mean'] = local_rolling.mean()\n",
    "        df_return['rolling_min'] = local_rolling.min()\n",
    "        df_return['rolling_max'] = local_rolling.max()\n",
    "        df_return['general_rolling_mean'] = general_rolling.mean()\n",
    "        df_return['general_rolling_min'] = general_rolling.min()\n",
    "        df_return['general_rolling_max'] = general_rolling.max()\n",
    "        return df_return        \n",
    "    \n",
    "    def expanded_features(self, df, min_period = 2):\n",
    "        expanding = df['Value'].expanding(min_period, center = True)\n",
    "        pd_return = pd.DataFrame()\n",
    "        pd_return['expanding_mean'] = expanding.mean()\n",
    "        pd_return['expanding_max'] = expanding.max()\n",
    "        pd_return['expanding_min'] = expanding.min()\n",
    "        return pd_return\n",
    "\n",
    "    def all_features(self, df, fill0 = True):\n",
    "        df_return =  pd.concat(\n",
    "            [\n",
    "                self.time_features(df),\n",
    "                self.lag_features(df),\n",
    "                self.rolling_features(df),\n",
    "                self.expanded_features(df),\n",
    "                df[['League']]\n",
    "            ],\n",
    "            \n",
    "            axis = 1\n",
    "        )\n",
    "        return df_return.fillna(0) if fill0 else df_return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "F:\\AppSSD\\WorkTools\\Anaconda\\envs\\gpu-ML\\lib\\site-packages\\ipykernel_launcher.py:51: FutureWarning: The `center` argument on `expanding` will be removed in the future\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>LeagueDay</th>\n",
       "      <th>day_of_week</th>\n",
       "      <th>month</th>\n",
       "      <th>y_lag_1</th>\n",
       "      <th>y_lag_2</th>\n",
       "      <th>y_lag_3</th>\n",
       "      <th>y_lag_4</th>\n",
       "      <th>rolling_mean</th>\n",
       "      <th>rolling_min</th>\n",
       "      <th>rolling_max</th>\n",
       "      <th>general_rolling_mean</th>\n",
       "      <th>general_rolling_min</th>\n",
       "      <th>general_rolling_max</th>\n",
       "      <th>expanding_mean</th>\n",
       "      <th>expanding_max</th>\n",
       "      <th>expanding_min</th>\n",
       "      <th>League</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Date</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2016-09-08</th>\n",
       "      <td>1.0</td>\n",
       "      <td>3</td>\n",
       "      <td>9</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Essence</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            LeagueDay  day_of_week  month  y_lag_1  y_lag_2  y_lag_3  y_lag_4  \\\n",
       "Date                                                                            \n",
       "2016-09-08        1.0            3      9      0.0      0.0      0.0      0.0   \n",
       "\n",
       "            rolling_mean  rolling_min  rolling_max  general_rolling_mean  \\\n",
       "Date                                                                       \n",
       "2016-09-08           0.0          0.0          0.0                   0.0   \n",
       "\n",
       "            general_rolling_min  general_rolling_max  expanding_mean  \\\n",
       "Date                                                                   \n",
       "2016-09-08                  0.0                  0.0             0.0   \n",
       "\n",
       "            expanding_max  expanding_min   League  \n",
       "Date                                               \n",
       "2016-09-08            0.0            0.0  Essence  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>y_step_1</th>\n",
       "      <th>y_step_2</th>\n",
       "      <th>y_step_3</th>\n",
       "      <th>y_step_4</th>\n",
       "      <th>y_step_5</th>\n",
       "      <th>League</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Date</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2016-09-08</th>\n",
       "      <td>64.51451</td>\n",
       "      <td>65.0028</td>\n",
       "      <td>65.05764</td>\n",
       "      <td>65.98811</td>\n",
       "      <td>66.01724</td>\n",
       "      <td>Essence</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            y_step_1  y_step_2  y_step_3  y_step_4  y_step_5   League\n",
       "Date                                                                 \n",
       "2016-09-08  64.51451   65.0028  65.05764  65.98811  66.01724  Essence"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "feature_engineer = FeatureEngineer()\n",
    "\n",
    "# X_train = feature_engineer.all_features(df_train)\n",
    "# y_train = feature_engineer.target(df_train)\n",
    "# X_val = feature_engineer.all_features(df_val)\n",
    "# y_val = feature_engineer.target(df_val)\n",
    "X = feature_engineer.all_features(df)\n",
    "y = feature_engineer.target(df)\n",
    "\n",
    "display(X.head(1))\n",
    "display(y.head(1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_val_split(df):\n",
    "    is_last_league = df['League'] == Constants.LEAGUES[-1]\n",
    "\n",
    "    df_train = df[~is_last_league].drop('League', axis = 1)\n",
    "    df_val = df[is_last_league].drop('League', axis = 1)\n",
    "    return df_train, df_val\n",
    "def print_train_val_stats(train,val):\n",
    "    print('shapes: ', train.shape, val.shape)\n",
    "    display(train.head(2))\n",
    "    display(val.head(2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shapes:  (1719, 16) (88, 16)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>LeagueDay</th>\n",
       "      <th>day_of_week</th>\n",
       "      <th>month</th>\n",
       "      <th>y_lag_1</th>\n",
       "      <th>y_lag_2</th>\n",
       "      <th>y_lag_3</th>\n",
       "      <th>y_lag_4</th>\n",
       "      <th>rolling_mean</th>\n",
       "      <th>rolling_min</th>\n",
       "      <th>rolling_max</th>\n",
       "      <th>general_rolling_mean</th>\n",
       "      <th>general_rolling_min</th>\n",
       "      <th>general_rolling_max</th>\n",
       "      <th>expanding_mean</th>\n",
       "      <th>expanding_max</th>\n",
       "      <th>expanding_min</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Date</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2016-09-08</th>\n",
       "      <td>1.0</td>\n",
       "      <td>3</td>\n",
       "      <td>9</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2016-09-09</th>\n",
       "      <td>2.0</td>\n",
       "      <td>4</td>\n",
       "      <td>9</td>\n",
       "      <td>64.00417</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>64.25934</td>\n",
       "      <td>64.51451</td>\n",
       "      <td>64.00417</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            LeagueDay  day_of_week  month   y_lag_1  y_lag_2  y_lag_3  \\\n",
       "Date                                                                    \n",
       "2016-09-08        1.0            3      9   0.00000      0.0      0.0   \n",
       "2016-09-09        2.0            4      9  64.00417      0.0      0.0   \n",
       "\n",
       "            y_lag_4  rolling_mean  rolling_min  rolling_max  \\\n",
       "Date                                                          \n",
       "2016-09-08      0.0           0.0          0.0          0.0   \n",
       "2016-09-09      0.0           0.0          0.0          0.0   \n",
       "\n",
       "            general_rolling_mean  general_rolling_min  general_rolling_max  \\\n",
       "Date                                                                         \n",
       "2016-09-08                   0.0                  0.0                  0.0   \n",
       "2016-09-09                   0.0                  0.0                  0.0   \n",
       "\n",
       "            expanding_mean  expanding_max  expanding_min  \n",
       "Date                                                      \n",
       "2016-09-08         0.00000        0.00000        0.00000  \n",
       "2016-09-09        64.25934       64.51451       64.00417  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>LeagueDay</th>\n",
       "      <th>day_of_week</th>\n",
       "      <th>month</th>\n",
       "      <th>y_lag_1</th>\n",
       "      <th>y_lag_2</th>\n",
       "      <th>y_lag_3</th>\n",
       "      <th>y_lag_4</th>\n",
       "      <th>rolling_mean</th>\n",
       "      <th>rolling_min</th>\n",
       "      <th>rolling_max</th>\n",
       "      <th>general_rolling_mean</th>\n",
       "      <th>general_rolling_min</th>\n",
       "      <th>general_rolling_max</th>\n",
       "      <th>expanding_mean</th>\n",
       "      <th>expanding_max</th>\n",
       "      <th>expanding_min</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Date</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2021-07-23</th>\n",
       "      <td>1.0</td>\n",
       "      <td>4</td>\n",
       "      <td>7</td>\n",
       "      <td>104.50518</td>\n",
       "      <td>105.00000</td>\n",
       "      <td>105.6665</td>\n",
       "      <td>105.89479</td>\n",
       "      <td>78.817024</td>\n",
       "      <td>50.0</td>\n",
       "      <td>105.00000</td>\n",
       "      <td>87.189063</td>\n",
       "      <td>50.0</td>\n",
       "      <td>106.27503</td>\n",
       "      <td>111.783670</td>\n",
       "      <td>205.0</td>\n",
       "      <td>18.01531</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-07-24</th>\n",
       "      <td>2.0</td>\n",
       "      <td>5</td>\n",
       "      <td>7</td>\n",
       "      <td>50.00000</td>\n",
       "      <td>104.50518</td>\n",
       "      <td>105.0000</td>\n",
       "      <td>105.66650</td>\n",
       "      <td>73.217024</td>\n",
       "      <td>50.0</td>\n",
       "      <td>104.50518</td>\n",
       "      <td>86.312877</td>\n",
       "      <td>50.0</td>\n",
       "      <td>106.00000</td>\n",
       "      <td>111.754743</td>\n",
       "      <td>205.0</td>\n",
       "      <td>18.01531</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            LeagueDay  day_of_week  month    y_lag_1    y_lag_2   y_lag_3  \\\n",
       "Date                                                                        \n",
       "2021-07-23        1.0            4      7  104.50518  105.00000  105.6665   \n",
       "2021-07-24        2.0            5      7   50.00000  104.50518  105.0000   \n",
       "\n",
       "              y_lag_4  rolling_mean  rolling_min  rolling_max  \\\n",
       "Date                                                            \n",
       "2021-07-23  105.89479     78.817024         50.0    105.00000   \n",
       "2021-07-24  105.66650     73.217024         50.0    104.50518   \n",
       "\n",
       "            general_rolling_mean  general_rolling_min  general_rolling_max  \\\n",
       "Date                                                                         \n",
       "2021-07-23             87.189063                 50.0            106.27503   \n",
       "2021-07-24             86.312877                 50.0            106.00000   \n",
       "\n",
       "            expanding_mean  expanding_max  expanding_min  \n",
       "Date                                                      \n",
       "2021-07-23      111.783670          205.0       18.01531  \n",
       "2021-07-24      111.754743          205.0       18.01531  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shapes:  (1719, 5) (88, 5)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>y_step_1</th>\n",
       "      <th>y_step_2</th>\n",
       "      <th>y_step_3</th>\n",
       "      <th>y_step_4</th>\n",
       "      <th>y_step_5</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Date</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2016-09-08</th>\n",
       "      <td>64.51451</td>\n",
       "      <td>65.00280</td>\n",
       "      <td>65.05764</td>\n",
       "      <td>65.98811</td>\n",
       "      <td>66.01724</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2016-09-09</th>\n",
       "      <td>65.00280</td>\n",
       "      <td>65.05764</td>\n",
       "      <td>65.98811</td>\n",
       "      <td>66.01724</td>\n",
       "      <td>69.08462</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            y_step_1  y_step_2  y_step_3  y_step_4  y_step_5\n",
       "Date                                                        \n",
       "2016-09-08  64.51451  65.00280  65.05764  65.98811  66.01724\n",
       "2016-09-09  65.00280  65.05764  65.98811  66.01724  69.08462"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>y_step_1</th>\n",
       "      <th>y_step_2</th>\n",
       "      <th>y_step_3</th>\n",
       "      <th>y_step_4</th>\n",
       "      <th>y_step_5</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Date</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2021-07-23</th>\n",
       "      <td>62.00000</td>\n",
       "      <td>72.57994</td>\n",
       "      <td>77.0000</td>\n",
       "      <td>75.9145</td>\n",
       "      <td>75.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-07-24</th>\n",
       "      <td>72.57994</td>\n",
       "      <td>77.00000</td>\n",
       "      <td>75.9145</td>\n",
       "      <td>75.0000</td>\n",
       "      <td>76.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            y_step_1  y_step_2  y_step_3  y_step_4  y_step_5\n",
       "Date                                                        \n",
       "2021-07-23  62.00000  72.57994   77.0000   75.9145      75.0\n",
       "2021-07-24  72.57994  77.00000   75.9145   75.0000      76.0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "X_train, X_val = train_val_split(X)\n",
    "y_train, y_val = train_val_split(y)\n",
    "\n",
    "print_train_val_stats(X_train, X_val)\n",
    "print_train_val_stats(y_train, y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving ./data-training/X_train.csv\n",
      "Saving ./data-training/X_val.csv\n",
      "Saving ./data-training/y_train.csv\n",
      "Saving ./data-training/y_val.csv\n"
     ]
    }
   ],
   "source": [
    "out_names = ['X_train', 'X_val', 'y_train', 'y_val']\n",
    "outs_dfs = [X_train, X_val, y_train, y_val]\n",
    "for name, df in zip(out_names, outs_dfs):\n",
    "    out_name = f'./data-training/{name}.csv'\n",
    "    print(f'Saving {out_name}')\n",
    "    df.reset_index().to_csv(out_name, index=False)\n"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "dbd5bea911370489c6458bc4dc1ddafddd410f0ca74aced470acd31061c2c4f6"
  },
  "kernelspec": {
   "display_name": "Python 3.7.11 ('gpu-ML')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.11"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}