Regime Forecasting - II.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Regime Forecasting II\n",
    "\n",
    "Preparation of the dataset for recession forecasting. Cleaning of the data and feature selection to reduce the number of variables in the data.\n",
    "\n",
    "## Table of Contents:\n",
    "&nbsp;&nbsp;1. [Set Up Environment and Read Data](#1)\n",
    "\n",
    "&nbsp;&nbsp;2. [Data Cleaning](#2)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Set Up Environment and Read Data <a id=\"1\"></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#load libraries\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt \n",
    "\n",
    "from statsmodels.tsa.stattools import adfuller #to check unit root in time series \n",
    "from sklearn import preprocessing\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "\n",
    "import seaborn as sns #for correlation heatmap\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>Regime</th>\n",
       "      <th>RPI</th>\n",
       "      <th>W875RX1</th>\n",
       "      <th>DPCERA3M086SBEA</th>\n",
       "      <th>CMRMTSPLx</th>\n",
       "      <th>RETAILx</th>\n",
       "      <th>INDPRO</th>\n",
       "      <th>IPFPNSS</th>\n",
       "      <th>IPFINAL</th>\n",
       "      <th>...</th>\n",
       "      <th>DSERRG3M086SBEA</th>\n",
       "      <th>CES0600000008</th>\n",
       "      <th>CES2000000008</th>\n",
       "      <th>CES3000000008</th>\n",
       "      <th>UMCSENTx</th>\n",
       "      <th>MZMSL</th>\n",
       "      <th>DTCOLNVHFNM</th>\n",
       "      <th>DTCTHFNM</th>\n",
       "      <th>INVEST</th>\n",
       "      <th>VXOCLSx</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1/1/59</td>\n",
       "      <td>Normal</td>\n",
       "      <td>2437.296</td>\n",
       "      <td>2288.8</td>\n",
       "      <td>17.302</td>\n",
       "      <td>292258.8329</td>\n",
       "      <td>18235.77392</td>\n",
       "      <td>22.6248</td>\n",
       "      <td>23.4555</td>\n",
       "      <td>22.1893</td>\n",
       "      <td>...</td>\n",
       "      <td>11.358</td>\n",
       "      <td>2.13</td>\n",
       "      <td>2.45</td>\n",
       "      <td>2.04</td>\n",
       "      <td>NaN</td>\n",
       "      <td>274.9</td>\n",
       "      <td>6476.0</td>\n",
       "      <td>12298.0</td>\n",
       "      <td>84.2043</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2/1/59</td>\n",
       "      <td>Normal</td>\n",
       "      <td>2446.902</td>\n",
       "      <td>2297.0</td>\n",
       "      <td>17.482</td>\n",
       "      <td>294429.5453</td>\n",
       "      <td>18369.56308</td>\n",
       "      <td>23.0679</td>\n",
       "      <td>23.7720</td>\n",
       "      <td>22.3816</td>\n",
       "      <td>...</td>\n",
       "      <td>11.375</td>\n",
       "      <td>2.14</td>\n",
       "      <td>2.46</td>\n",
       "      <td>2.05</td>\n",
       "      <td>NaN</td>\n",
       "      <td>276.0</td>\n",
       "      <td>6476.0</td>\n",
       "      <td>12298.0</td>\n",
       "      <td>83.5280</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3/1/59</td>\n",
       "      <td>Normal</td>\n",
       "      <td>2462.689</td>\n",
       "      <td>2314.0</td>\n",
       "      <td>17.647</td>\n",
       "      <td>293425.3813</td>\n",
       "      <td>18523.05762</td>\n",
       "      <td>23.4002</td>\n",
       "      <td>23.9159</td>\n",
       "      <td>22.4914</td>\n",
       "      <td>...</td>\n",
       "      <td>11.395</td>\n",
       "      <td>2.15</td>\n",
       "      <td>2.45</td>\n",
       "      <td>2.07</td>\n",
       "      <td>NaN</td>\n",
       "      <td>277.4</td>\n",
       "      <td>6508.0</td>\n",
       "      <td>12349.0</td>\n",
       "      <td>81.6405</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4/1/59</td>\n",
       "      <td>Normal</td>\n",
       "      <td>2478.744</td>\n",
       "      <td>2330.3</td>\n",
       "      <td>17.584</td>\n",
       "      <td>299331.6505</td>\n",
       "      <td>18534.46600</td>\n",
       "      <td>23.8987</td>\n",
       "      <td>24.2613</td>\n",
       "      <td>22.8210</td>\n",
       "      <td>...</td>\n",
       "      <td>11.436</td>\n",
       "      <td>2.16</td>\n",
       "      <td>2.47</td>\n",
       "      <td>2.08</td>\n",
       "      <td>NaN</td>\n",
       "      <td>278.1</td>\n",
       "      <td>6620.0</td>\n",
       "      <td>12484.0</td>\n",
       "      <td>81.8099</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5/1/59</td>\n",
       "      <td>Normal</td>\n",
       "      <td>2493.228</td>\n",
       "      <td>2345.8</td>\n",
       "      <td>17.796</td>\n",
       "      <td>301372.9597</td>\n",
       "      <td>18679.66354</td>\n",
       "      <td>24.2587</td>\n",
       "      <td>24.4628</td>\n",
       "      <td>23.0407</td>\n",
       "      <td>...</td>\n",
       "      <td>11.454</td>\n",
       "      <td>2.17</td>\n",
       "      <td>2.48</td>\n",
       "      <td>2.08</td>\n",
       "      <td>95.3</td>\n",
       "      <td>280.1</td>\n",
       "      <td>6753.0</td>\n",
       "      <td>12646.0</td>\n",
       "      <td>80.7315</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 130 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     Date  Regime       RPI  W875RX1  DPCERA3M086SBEA    CMRMTSPLx  \\\n",
       "0  1/1/59  Normal  2437.296   2288.8           17.302  292258.8329   \n",
       "1  2/1/59  Normal  2446.902   2297.0           17.482  294429.5453   \n",
       "2  3/1/59  Normal  2462.689   2314.0           17.647  293425.3813   \n",
       "3  4/1/59  Normal  2478.744   2330.3           17.584  299331.6505   \n",
       "4  5/1/59  Normal  2493.228   2345.8           17.796  301372.9597   \n",
       "\n",
       "       RETAILx   INDPRO  IPFPNSS  IPFINAL  ...  DSERRG3M086SBEA  \\\n",
       "0  18235.77392  22.6248  23.4555  22.1893  ...           11.358   \n",
       "1  18369.56308  23.0679  23.7720  22.3816  ...           11.375   \n",
       "2  18523.05762  23.4002  23.9159  22.4914  ...           11.395   \n",
       "3  18534.46600  23.8987  24.2613  22.8210  ...           11.436   \n",
       "4  18679.66354  24.2587  24.4628  23.0407  ...           11.454   \n",
       "\n",
       "   CES0600000008  CES2000000008  CES3000000008  UMCSENTx  MZMSL  DTCOLNVHFNM  \\\n",
       "0           2.13           2.45           2.04       NaN  274.9       6476.0   \n",
       "1           2.14           2.46           2.05       NaN  276.0       6476.0   \n",
       "2           2.15           2.45           2.07       NaN  277.4       6508.0   \n",
       "3           2.16           2.47           2.08       NaN  278.1       6620.0   \n",
       "4           2.17           2.48           2.08      95.3  280.1       6753.0   \n",
       "\n",
       "   DTCTHFNM   INVEST  VXOCLSx  \n",
       "0   12298.0  84.2043      NaN  \n",
       "1   12298.0  83.5280      NaN  \n",
       "2   12349.0  81.6405      NaN  \n",
       "3   12484.0  81.8099      NaN  \n",
       "4   12646.0  80.7315      NaN  \n",
       "\n",
       "[5 rows x 130 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bigmacro=pd.read_csv(\"Macroeconomic_Variables.csv\")\n",
    "bigmacro=bigmacro.rename(columns={'sasdate':'Date'})\n",
    "Recession_periods=pd.read_csv('Recession_Periods.csv')\n",
    "bigmacro.insert(loc=1,column=\"Regime\", value=Recession_periods['Regime'].values)\n",
    "bigmacro.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Data Cleaning <a id=\"2\"></a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will follow the steps below to clean data and make it ready for feature selection process.\n",
    "\n",
    "1. Remove the variables with missing observations\n",
    "2. Add lags of the variables as additional features\n",
    "3. Test stationarity of time series\n",
    "4. Standardize the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PERMIT:13\n",
      "PERMITNE:13\n",
      "PERMITMW:13\n",
      "PERMITS:13\n",
      "PERMITW:13\n",
      "ACOGNO:398\n",
      "ANDENOx:110\n",
      "TWEXMMTH:168\n",
      "UMCSENTx:155\n",
      "VXOCLSx:42\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(718, 120)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#remove columns with missing observations\n",
    "missing_colnames=[]\n",
    "for i in bigmacro.drop(['Date','Regime'],axis=1):\n",
    "    observations=len(bigmacro)-bigmacro[i].count()\n",
    "    if (observations>10):\n",
    "        print(i+':'+str(observations))\n",
    "        missing_colnames.append(i)\n",
    " \n",
    "bigmacro=bigmacro.drop(labels=missing_colnames, axis=1)\n",
    "\n",
    "#rows with missing values\n",
    "bigmacro=bigmacro.dropna(axis=0)\n",
    "\n",
    "bigmacro.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add lags\n",
    "for col in bigmacro.drop(['Date', 'Regime'], axis=1):\n",
    "    for n in [3,6,9,12,18]:\n",
    "        bigmacro['{} {}M lag'.format(col, n)] = bigmacro[col].shift(n).ffill().values\n",
    "\n",
    "# 1 month ahead prediction\n",
    "bigmacro[\"Regime\"]=bigmacro[\"Regime\"].shift(-1)\n",
    "\n",
    "bigmacro=bigmacro.dropna(axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(699, 710)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bigmacro.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Augmented Dickey-Fuller Test can be used to test for stationarity in macroeconomic time series variables. We will use `adfuller` function from `statsmodels` module in Python. More information about the function can be found __[here](https://www.statsmodels.org/dev/generated/statsmodels.tsa.stattools.adfuller.html)__."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#check stationarity\n",
    "from statsmodels.tsa.stattools import adfuller #to check unit root in time series \n",
    "threshold=0.01 #significance level\n",
    "for column in bigmacro.drop(['Date','Regime'], axis=1):\n",
    "    result=adfuller(bigmacro[column])\n",
    "    if result[1]>threshold:\n",
    "        bigmacro[column]=bigmacro[column].diff()\n",
    "bigmacro=bigmacro.dropna(axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "threshold=0.01 #significance level\n",
    "for column in bigmacro.drop(['Date','Regime'], axis=1):\n",
    "    result=adfuller(bigmacro[column])\n",
    "    if result[1]>threshold:\n",
    "        bigmacro[column]=bigmacro[column].diff()\n",
    "bigmacro=bigmacro.dropna(axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CES0600000008 6M lag\n"
     ]
    }
   ],
   "source": [
    "threshold=0.01 #significance level\n",
    "for column in bigmacro.drop(['Date','Regime'], axis=1):\n",
    "    result=adfuller(bigmacro[column])\n",
    "    if result[1]>threshold:\n",
    "        print(column)\n",
    "bigmacro=bigmacro.dropna(axis=0)      "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(697, 710)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Standardize\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "features=bigmacro.drop(['Date','Regime'],axis=1)\n",
    "col_names=features.columns\n",
    "\n",
    "scaler=StandardScaler()\n",
    "scaler.fit(features)\n",
    "standardized_features=scaler.transform(features)\n",
    "standardized_features.shape\n",
    "df=pd.DataFrame(data=standardized_features,columns=col_names)\n",
    "df.insert(loc=0,column=\"Date\", value=bigmacro['Date'].values)\n",
    "df.insert(loc=1,column='Regime', value=bigmacro['Regime'].values)\n",
    "df.head()\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(\"Dataset_Cleaned.csv\", index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}