3_TFT_Individual.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "ae8d2fb9",
   "metadata": {},
   "source": [
    "# Temporal Fusion Transformer - Individual\n",
    "### Load packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f4787444",
   "metadata": {},
   "outputs": [],
   "source": [
    "import lightning.pytorch as pl\n",
    "import time as time\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import torch\n",
    "import os\n",
    "\n",
    "from pytorch_forecasting.models.temporal_fusion_transformer.tuning import optimize_hyperparameters\n",
    "from pytorch_forecasting import Baseline, TimeSeriesDataSet, TemporalFusionTransformer\n",
    "from pytorch_forecasting.data.encoders import TorchNormalizer\n",
    "from pytorch_forecasting.metrics import QuantileLoss\n",
    "from lightning.pytorch.callbacks import EarlyStopping, LearningRateMonitor\n",
    "from lightning.pytorch.loggers import TensorBoardLogger\n",
    "from lightning.pytorch.tuner import Tuner"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "51d439ff",
   "metadata": {},
   "source": [
    "### Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "49308cf7",
   "metadata": {},
   "outputs": [],
   "source": [
    "Directory = 'C:/.../TFT_for_Stock_Movement_Prediction/data'\n",
    "\n",
    "# Target and return feature\n",
    "CCR = pd.read_csv(os.path.join(Directory, 'CCR.csv'), index_col = [0])\n",
    "\n",
    "### Features\n",
    "## Time features - Categorical\n",
    "time_features = pd.read_csv(os.path.join(Directory, 'time_features.csv'), index_col = [0])['0'].tolist()\n",
    "for i in range(len(time_features)):\n",
    "    locals()[time_features[i]] = pd.read_csv(os.path.join(Directory, time_features[i] + '.csv'), index_col = [0])\n",
    "\n",
    "## Basic historical features\n",
    "bh_features = pd.read_csv(os.path.join(Directory, 'bh_features.csv'), index_col = [0])['0'].tolist()\n",
    "for i in range(len(bh_features)):\n",
    "    locals()[bh_features[i]] = pd.read_csv(os.path.join(Directory, bh_features[i] + '.csv'), index_col = [0])\n",
    "\n",
    "# Categorical\n",
    "bh_categorical_features = pd.read_csv(os.path.join(Directory, 'bh_categorical_features.csv'), index_col = [0])['0'].tolist()\n",
    "\n",
    "# Continuous\n",
    "bh_continuous_features = pd.read_csv(os.path.join(Directory, 'bh_continuous_features.csv'), index_col = [0])['0'].tolist()\n",
    "\n",
    "## Technical indicators - Continuous\n",
    "indicator_features = pd.read_csv(os.path.join(Directory, 'indicator_features.csv'), index_col = [0])['0'].tolist()\n",
    "for i in range(len(indicator_features)):\n",
    "    locals()[indicator_features[i]] = pd.read_csv(os.path.join(Directory, indicator_features[i] + '.csv'), index_col = [0])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6dc4b29f",
   "metadata": {},
   "source": [
    "### Model preparation\n",
    "#### Define variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "45e8868b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Study periods length\n",
    "period_b = 0, 250, 500, 750, 1000, 1250, 1500, 1750, 2000, 2250\n",
    "period_e = 1000, 1250, 1500, 1750, 2000, 2250, 2500, 2750, 3000, 3250\n",
    "\n",
    "# Split period into training, validation and test set\n",
    "training_size = 750\n",
    "test_size = 250\n",
    "validation_split = 0.2\n",
    "training_cutoff = int(training_size - training_size * validation_split)\n",
    "\n",
    "# Target\n",
    "Target_feature = ['CCR']\n",
    "\n",
    "# Features\n",
    "Feature_type = ['object']\n",
    "time_varying_known_categoricals = time_features\n",
    "time_varying_unknown_categoricals = bh_categorical_features\n",
    "time_varying_unknown_reals = bh_continuous_features + indicator_features\n",
    "\n",
    "## Model parameters\n",
    "# Dataset\n",
    "max_prediction_length = 1\n",
    "max_encoder_length = 258\n",
    "\n",
    "# Data loader\n",
    "batch_size = 64\n",
    "\n",
    "# Trainer\n",
    "max_epochs = 100\n",
    "gradient_clip_val = 1\n",
    "log_every_n_steps = 10\n",
    "\n",
    "# TFT\n",
    "learning_rate = 0.001\n",
    "hidden_size = 50\n",
    "hidden_continuous_size = 25\n",
    "attention_head_size = 4\n",
    "dropout = 0.20\n",
    "quantiles = [0.1, 0.5, 0.9]\n",
    "output_size = len(quantiles)\n",
    "loss = QuantileLoss(quantiles = quantiles)\n",
    "\n",
    "# Early stopping\n",
    "monitor = 'val_loss'\n",
    "min_delta = 1e-4\n",
    "patience = 10\n",
    "mode = 'min'\n",
    "\n",
    "# Learning rate logger\n",
    "lr_logger = LearningRateMonitor()\n",
    "\n",
    "# Learning rate finder\n",
    "max_lr = 1\n",
    "min_lr = 1e-6\n",
    "\n",
    "# Hyperparameter tuning\n",
    "batch_size_list = [16, 32, 64, 128]\n",
    "\n",
    "# File path to save results\n",
    "File_name_results = 'results/Individual/Results_Individual.csv'\n",
    "File_name_results_p90 = 'results/Individual/Results_p90_Individual.csv'\n",
    "File_name_results_p10 = 'results/Individual/Results_p10_Individual.csv'\n",
    "File_name_baseline = 'results/Baseline/Baseline.csv'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5a6acc57",
   "metadata": {},
   "source": [
    "#### Preparation of datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8153e4f7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Datasets for each study period and stock\n",
    "def dataset(period, stock):\n",
    "    data = pd.DataFrame(index = globals()[Target_feature[0]].index[period_b[period] : period_e[period]])\n",
    "    \n",
    "    ## Add features\n",
    "    data['Time_idx'] = range(period_b[0] + 1, period_e[0] + 1)\n",
    "    data['Target'] = globals()[Target_feature[0]][[globals()[Target_feature[0]].columns[stock]]][period_b[period] : period_e[period]]\n",
    "    data['Stock'] = globals()[Target_feature[0]].columns[stock]\n",
    "    \n",
    "    # Time varying known categoricals\n",
    "    for f in range(len(time_varying_known_categoricals)):\n",
    "        data[time_varying_known_categoricals[f]] = globals()[time_varying_known_categoricals[f]].astype(Feature_type[0])\n",
    "    \n",
    "    # Time varying unknown categoricals\n",
    "    for f in range(len(time_varying_unknown_categoricals)):\n",
    "        data[time_varying_unknown_categoricals[f]] = globals()[time_varying_unknown_categoricals[f]][globals()[time_varying_unknown_categoricals[f]].columns[stock]].astype(Feature_type[0])\n",
    "    \n",
    "    # Time varying unknown reals\n",
    "    for f in range(len(time_varying_unknown_reals)):\n",
    "        data[time_varying_unknown_reals[f]] = globals()[time_varying_unknown_reals[f]][globals()[time_varying_unknown_reals[f]].columns[stock]]\n",
    "    return data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "44982cc0",
   "metadata": {},
   "source": [
    "### Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c9fabc37",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "start = time.time()\n",
    "Results = pd.DataFrame(index = range(period_b[0], period_e[9] - training_size), columns = locals()[Target_feature[0]].columns)\n",
    "Results_p90 = pd.DataFrame(index = range(period_b[0], period_e[9] - training_size), columns = locals()[Target_feature[0]].columns)\n",
    "Results_p10 = pd.DataFrame(index = range(period_b[0], period_e[9] - training_size), columns = locals()[Target_feature[0]].columns)\n",
    "Baseline_results = pd.DataFrame(index = range(period_b[0], period_e[9] - training_size), columns = locals()[Target_feature[0]].columns)\n",
    "for i in range(len(period_b)):\n",
    "    start_period = time.time()\n",
    "    for j in range(len(locals()[Target_feature[0]].columns)):\n",
    "        start_stock = time.time()\n",
    "        data = dataset(i, j)\n",
    "        \n",
    "        #### Create dataframe\n",
    "        training = TimeSeriesDataSet(\n",
    "        data[lambda x: x.Time_idx <= training_cutoff],\n",
    "        time_idx = data.columns[0],\n",
    "        target = data.columns[1],\n",
    "        group_ids = [data.columns[2]],\n",
    "        max_encoder_length = max_encoder_length,\n",
    "        max_prediction_length = max_prediction_length,\n",
    "        time_varying_known_reals = [data.columns[0]],\n",
    "        time_varying_unknown_reals = [data.columns[1]] + time_varying_unknown_reals,\n",
    "        time_varying_known_categoricals = time_varying_known_categoricals,\n",
    "        time_varying_unknown_categoricals = time_varying_unknown_categoricals,\n",
    "        target_normalizer = TorchNormalizer())\n",
    "        validation = TimeSeriesDataSet.from_dataset(training, data[lambda x: x.Time_idx <= training_size], min_prediction_idx = training_cutoff + 1)\n",
    "        test = TimeSeriesDataSet.from_dataset(training, data, min_prediction_idx = training_size + 1)\n",
    "        train_dataloader = training.to_dataloader(train = True, batch_size = batch_size)\n",
    "        val_dataloader = validation.to_dataloader(train = False, batch_size = training_size - training_cutoff)\n",
    "        \n",
    "        #### Model architecture\n",
    "        early_stop_callback = EarlyStopping(monitor = monitor, min_delta = min_delta, verbose = True, patience = patience, mode = mode)\n",
    "        logger = TensorBoardLogger('logs', name = f'Individual/Period_{i + 1}/Stock_{j + 1}')\n",
    "        trainer = pl.Trainer(\n",
    "        max_epochs = max_epochs,\n",
    "        enable_model_summary = False,\n",
    "        gradient_clip_val = gradient_clip_val,\n",
    "        log_every_n_steps = log_every_n_steps,\n",
    "        callbacks = [lr_logger, early_stop_callback],\n",
    "        logger = logger)\n",
    "        tft = TemporalFusionTransformer.from_dataset(\n",
    "        training,\n",
    "        learning_rate = learning_rate,                        \n",
    "        hidden_size = hidden_size,\n",
    "        hidden_continuous_size = hidden_continuous_size,\n",
    "        attention_head_size = attention_head_size,                     \n",
    "        dropout = dropout,\n",
    "        output_size = output_size,\n",
    "        loss = loss)\n",
    "        trainer.fit(tft, train_dataloaders = train_dataloader, val_dataloaders = val_dataloader)\n",
    "        \n",
    "        #### Predicting\n",
    "        best_tft = TemporalFusionTransformer.load_from_checkpoint(trainer.checkpoint_callback.best_model_path)\n",
    "        predictions = pd.DataFrame(best_tft.predict(test).cpu())\n",
    "        Results.iloc[period_b[i] : period_b[i] + test_size, j] = predictions[0]\n",
    "        predictions_p90 = pd.DataFrame(best_tft.predict(test, mode = 'quantiles').cpu()[:,0][:,2])\n",
    "        Results_p90.iloc[period_b[i] : period_b[i] + test_size, j] = predictions_p90[0]\n",
    "        predictions_p10 = pd.DataFrame(best_tft.predict(test, mode = 'quantiles').cpu()[:,0][:,0])\n",
    "        Results_p10.iloc[period_b[i] : period_b[i] + test_size, j] = predictions_p10[0]\n",
    "        baseline_predictions = pd.DataFrame(Baseline().predict(test).cpu())\n",
    "        Baseline_results.iloc[period_b[i] : period_b[i] + test_size, j] = baseline_predictions[0]\n",
    "        print(f'Compilation time - Period {i + 1} - Stock {j + 1}: {round(time.time() - start_stock)} seconds')\n",
    "    print(f'Compilation time - Period {i + 1}: {round(time.time() - start_period)} seconds')\n",
    "Results.to_csv(File_name_results)\n",
    "Results_p90.to_csv(File_name_results_p90)\n",
    "Results_p10.to_csv(File_name_results_p10)\n",
    "Baseline_results.to_csv(File_name_baseline)\n",
    "print(f'Compilation time: {round(time.time() - start)} seconds')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "059adff5",
   "metadata": {},
   "source": [
    "### Learningrate finder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "91722a9e",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Create dataframe\n",
    "data = dataset(np.random.randint(10), np.random.randint(33))\n",
    "training = TimeSeriesDataSet(\n",
    "data[lambda x: x.Time_idx <= training_cutoff],\n",
    "time_idx = data.columns[0],\n",
    "target = data.columns[1],\n",
    "group_ids = [data.columns[2]],\n",
    "max_encoder_length = max_encoder_length,\n",
    "max_prediction_length = max_prediction_length,\n",
    "time_varying_known_reals = [data.columns[0]],\n",
    "time_varying_unknown_reals = [data.columns[1]] + time_varying_unknown_reals,\n",
    "time_varying_known_categoricals = time_varying_known_categoricals,\n",
    "time_varying_unknown_categoricals = time_varying_unknown_categoricals,\n",
    "target_normalizer = TorchNormalizer())\n",
    "validation = TimeSeriesDataSet.from_dataset(training, data[lambda x: x.Time_idx <= training_size], min_prediction_idx = training_cutoff + 1)\n",
    "test = TimeSeriesDataSet.from_dataset(training, data, min_prediction_idx = training_size + 1)\n",
    "train_dataloader = training.to_dataloader(train = True, batch_size = batch_size)\n",
    "val_dataloader = validation.to_dataloader(train = False, batch_size = training_size - training_cutoff)\n",
    "\n",
    "# Run learningrate finder model\n",
    "trainer = pl.Trainer(gradient_clip_val = gradient_clip_val)\n",
    "tft = TemporalFusionTransformer.from_dataset(\n",
    "    training,\n",
    "    hidden_size = hidden_size,\n",
    "    hidden_continuous_size = hidden_continuous_size,\n",
    "    attention_head_size = attention_head_size,\n",
    "    dropout = dropout,\n",
    "    output_size = output_size,\n",
    "    loss = loss)\n",
    "res = Tuner(trainer).lr_find(\n",
    "    tft,\n",
    "    train_dataloaders=train_dataloader,\n",
    "    val_dataloaders=val_dataloader,\n",
    "    max_lr = max_lr,\n",
    "    min_lr = min_lr)\n",
    "print(f'suggested learning rate: {res.suggestion()}')\n",
    "fig = res.plot(show = True, suggest = True)\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "82f2bae5",
   "metadata": {},
   "source": [
    "### Hyperparameter tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "35b15f8b",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "start = time.time()\n",
    "list_values, list_parameters = [], []\n",
    "\n",
    "# Create dataframe\n",
    "data = dataset(np.random.randint(10), np.random.randint(33))\n",
    "training = TimeSeriesDataSet(\n",
    "data[lambda x: x.Time_idx <= training_cutoff],\n",
    "time_idx = data.columns[0],\n",
    "target = data.columns[1],\n",
    "group_ids = [data.columns[2]],\n",
    "max_encoder_length = max_encoder_length,\n",
    "max_prediction_length = max_prediction_length,\n",
    "time_varying_known_reals = [data.columns[0]],\n",
    "time_varying_unknown_reals = [data.columns[1]] + time_varying_unknown_reals,\n",
    "time_varying_known_categoricals = time_varying_known_categoricals,\n",
    "time_varying_unknown_categoricals = time_varying_unknown_categoricals,\n",
    "target_normalizer = TorchNormalizer())\n",
    "validation = TimeSeriesDataSet.from_dataset(training, data[lambda x: x.Time_idx <= training_size], min_prediction_idx = training_cutoff + 1)\n",
    "test = TimeSeriesDataSet.from_dataset(training, data, min_prediction_idx = training_size + 1)\n",
    "train_dataloader = training.to_dataloader(train = True, batch_size = batch_size)\n",
    "val_dataloader = validation.to_dataloader(train = False, batch_size = training_size - training_cutoff)\n",
    "\n",
    "# Run tuning model\n",
    "study = optimize_hyperparameters(\n",
    "    train_dataloader,\n",
    "    val_dataloader,\n",
    "    model_path = 'results/Hyperparameter tuning/Individual',\n",
    "    log_dir = 'results/Hyperparameter tuning/Individual',\n",
    "    n_trials = 10,\n",
    "    max_epochs = 100,\n",
    "    loss = loss,\n",
    "    output_size = output_size,\n",
    "    use_learning_rate_finder = False,\n",
    "    learning_rate_range = (0.001, 0.001),\n",
    "    trainer_kwargs = {'log_every_n_steps': log_every_n_steps, 'num_sanity_val_steps': 0})\n",
    "list_values.append(study.best_trial.value)\n",
    "list_parameters.append(study.best_trial.params)\n",
    "print(f'Compilation time: {round(time.time() - start)} seconds')\n",
    "print(*list_values, sep = '\\n')\n",
    "print(*list_parameters, sep = '\\n')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}