2b_Data_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "5ed04ecc",
   "metadata": {},
   "source": [
    "# Data analysis\n",
    "### Load packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7d77d766",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import plotly.graph_objects as go\n",
    "import os\n",
    "\n",
    "from datetime import datetime\n",
    "from scipy.stats import skew\n",
    "from scipy.stats import kurtosis"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b2b744e8",
   "metadata": {},
   "source": [
    "### Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cc1e453f",
   "metadata": {},
   "outputs": [],
   "source": [
    "Directory = 'C:/.../TFT_for_Stock_Movement_Prediction/data'\n",
    "\n",
    "# Target and return feature\n",
    "CCR = pd.read_csv(os.path.join(Directory, 'CCR.csv'), index_col = [0])\n",
    "\n",
    "# Portfolio\n",
    "constituents_data = pd.read_csv(os.path.join(Directory, 'constituents_data.csv'), index_col = [0], header = [0, 1])\n",
    "portfolio_table = pd.read_csv(os.path.join(Directory, 'portfolio_table.csv'), index_col = [0])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bb1d2010",
   "metadata": {},
   "source": [
    "### Define datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eb1b02a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove first unused dates\n",
    "constituents_data = constituents_data.iloc[1:, :]\n",
    "\n",
    "# Define study period lengths\n",
    "period_b = 0, 250, 500, 750, 1000, 1250, 1500, 1750, 2000, 2250\n",
    "period_e = 1000, 1250, 1500, 1750, 2000, 2250, 2500, 2750, 3000, 3250\n",
    "\n",
    "training_size = 750\n",
    "test_size = 250\n",
    "validation_split = 0.2\n",
    "training_cutoff = int(training_size - training_size * validation_split)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "da765f0c",
   "metadata": {},
   "source": [
    "### Plots\n",
    "#### Portfolio closing price"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c51330d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "portfolio_closing = pd.DataFrame(index = range(len(constituents_data.index)), columns = ['Close'])\n",
    "\n",
    "for i in range(len(constituents_data.index)):\n",
    "    portfolio_closing['Close'].iloc[i] = sum(constituents_data['Close'].iloc[i])\n",
    "\n",
    "fig = go.Figure(go.Scatter(name = 'Portfolio', line_width = 10, x = constituents_data.index, y = np.log(portfolio_closing['Close'].astype('float')), marker = dict(color = 'cadetblue', size = 2), showlegend = True))\n",
    "for i in range(len(period_e)):\n",
    "    fig.add_vline(x = datetime.strptime(constituents_data.index[period_e[i] - test_size], '%Y-%m-%d').timestamp() * 1000, line_width = 0.5, line_dash = '7', line_color = 'lightgrey', annotation_text = f'Test {i + 1}')  \n",
    "fig.add_vline(x = datetime.strptime(constituents_data.index[period_b[0] + training_cutoff], '%Y-%m-%d').timestamp() * 1000, line_width = 0.5, line_dash = '7', line_color = 'lightgrey', annotation_text = f'Val. 1')  \n",
    "fig.add_vline(x = datetime.strptime(constituents_data.index[period_b[0] + 3], '%Y-%m-%d').timestamp() * 1000, line_width = 0.5, line_dash = '7', line_color = 'lightgrey', annotation_text = f'Training 1')  \n",
    "fig.update_layout(annotations = [{**a, **{'y':1.02}} for a in fig.to_dict()['layout']['annotations']], yaxis_title = 'ln (closing price)', xaxis_title = 'Date', plot_bgcolor = 'white', xaxis_range = [constituents_data.index[0], constituents_data.index[-1]], showlegend = False)\n",
    "fig.update_xaxes(ticks = 'outside', showgrid = False, ticklen = 5)\n",
    "fig.update_yaxes(gridwidth = 0.5, gridcolor = 'lightgrey')\n",
    "config = {'toImageButtonOptions': {'scale': 10}}\n",
    "fig.show(config = config)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "113f2eb8",
   "metadata": {},
   "source": [
    "### Tables\n",
    "#### Portfolio summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "923daf66",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Portfolio sectors\n",
    "portfolio_sector = pd.DataFrame(index = portfolio_table['Industry'].value_counts().index, columns = ['Number of Stocks'])\n",
    "portfolio_sector['Number of Stocks'] = portfolio_table['Industry'].value_counts()\n",
    "\n",
    "# Mean return by sector\n",
    "stock_return = pd.DataFrame(index = [0], columns = constituents_data['Close'].columns)\n",
    "\n",
    "portfolio_return = pd.DataFrame(index = portfolio_table['Industry'].value_counts().index, columns = ['Mean return'])\n",
    "portfolio_return['Mean return'] = 0\n",
    "portfolio_return['Mean return'] = portfolio_return['Mean return'].astype('float')\n",
    "\n",
    "for i in constituents_data['Close'].columns: # For daily returns\n",
    "    stock_return[i] = sum(CCR[i]) / period_e[len(period_e) - 1]\n",
    "\n",
    "for i in range(len(portfolio_table['Industry'].value_counts().index)):\n",
    "    for j in range(len(stock_return.columns)):\n",
    "        if (portfolio_table['Industry'][portfolio_table['Symbol'] == stock_return.columns[j]][j] == portfolio_table['Industry'].value_counts().index[i]):\n",
    "            portfolio_return['Mean return'][i] += round(stock_return.iloc[0][j] / portfolio_table['Industry'].value_counts()[i] * 100, 4)\n",
    "\n",
    "# Standard deviation by sector\n",
    "portfolio_sd = pd.DataFrame(index = portfolio_table['Industry'].value_counts().index, columns = ['Standard deviation'])\n",
    "portfolio_sd['Standard deviation'] = 0\n",
    "portfolio_sd['Standard deviation'] = portfolio_sd['Standard deviation'].astype('float')\n",
    "\n",
    "for i in range(len(portfolio_table['Industry'].value_counts().index)):\n",
    "    temp = []\n",
    "    for j in range(len(stock_return.columns)):\n",
    "        if (portfolio_table['Industry'][portfolio_table['Symbol'] == stock_return.columns[j]][j] == portfolio_table['Industry'].value_counts().index[i]):\n",
    "            temp.append(stock_return.iloc[0][j])\n",
    "    portfolio_sd['Standard deviation'][i] = round(np.std(temp) * 100, 4)\n",
    "    \n",
    "# Skewness by sector\n",
    "portfolio_skew = pd.DataFrame(index = portfolio_table['Industry'].value_counts().index, columns = ['Skewness'])\n",
    "portfolio_skew['Skewness'] = 0\n",
    "portfolio_skew['Skewness'] = portfolio_skew['Skewness'].astype('float')\n",
    "\n",
    "for i in range(len(portfolio_table['Industry'].value_counts().index)):\n",
    "    temp = []\n",
    "    for j in range(len(stock_return.columns)):\n",
    "        if (portfolio_table['Industry'][portfolio_table['Symbol'] == stock_return.columns[j]][j] == portfolio_table['Industry'].value_counts().index[i]):\n",
    "            temp.append(stock_return.iloc[0][j])\n",
    "    portfolio_skew['Skewness'][i] = round(skew(temp), 4)\n",
    "    \n",
    "# Kurtosis by sector\n",
    "portfolio_kurtosis = pd.DataFrame(index = portfolio_table['Industry'].value_counts().index, columns = ['Kurtosis'])\n",
    "portfolio_kurtosis['Kurtosis'] = 0\n",
    "portfolio_kurtosis['Kurtosis'] = portfolio_kurtosis['Kurtosis'].astype('float')\n",
    "\n",
    "for i in range(len(portfolio_table['Industry'].value_counts().index)):\n",
    "    temp = []\n",
    "    for j in range(len(stock_return.columns)):\n",
    "        if (portfolio_table['Industry'][portfolio_table['Symbol'] == stock_return.columns[j]][j] == portfolio_table['Industry'].value_counts().index[i]):\n",
    "            temp.append(stock_return.iloc[0][j])\n",
    "    portfolio_kurtosis['Kurtosis'][i] = round(kurtosis(temp, fisher = False), 4)\n",
    "    \n",
    "# Portfolio table \n",
    "portfolio_statistics_table = pd.concat([portfolio_sector, portfolio_return, portfolio_sd, portfolio_skew, portfolio_kurtosis], axis=1)\n",
    "portfolio_statistics_table.loc['All'] = [len(portfolio_table), round(sum(stock_return.iloc[0]) / len(stock_return.iloc[0]) * 100, 4), round(np.std(stock_return.iloc[0]) * 100, 4), round(skew(stock_return.iloc[0]), 4), round(kurtosis(stock_return.iloc[0], fisher = False), 4)]\n",
    "portfolio_statistics_table['Number of Stocks'] = portfolio_statistics_table['Number of Stocks'].astype('int')\n",
    "portfolio_statistics_table"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}