u06-TimeGAN-non-stats.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "import pickle\n",
    "import warnings\n",
    "import itertools\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "from Soroosh_utilities import *\n",
    "\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastdtw import fastdtw\n",
    "from scipy.spatial.distance import euclidean\n",
    "\n",
    "# dtw, path = fastdtw(x, y, dist=euclidean)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "clone to the following repository \n",
    "\n",
    "https://github.com/vanderschaarlab/mlforhealthlabpub.git\n",
    "\n",
    "and set the path accordingly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"/home/soroosh/TimeGAN/\"\n",
    "sys.path.append(path)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ATTENTION PERMUTATION IS: False\n",
      "ATTENTION PERMUTATION IS: False\n",
      "ATTENTION PERMUTATION IS: False\n"
     ]
    }
   ],
   "source": [
    "## Necessary packages\n",
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "\n",
    "# 1. TimeGAN model\n",
    "from timegan import timegan\n",
    "# 2. Data loading\n",
    "from data_loading import real_data_loading, sine_data_generation\n",
    "# 3. Metrics\n",
    "from metrics.discriminative_metrics import discriminative_score_metrics\n",
    "from metrics.predictive_metrics import predictive_score_metrics\n",
    "from metrics.visualization_metrics import visualization\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepare_data_for_tgan(x, seq_length):\n",
    "    \n",
    "    dataX = []\n",
    "    \n",
    "    # Cut data by sequence length\n",
    "    for i in range(0, len(x) - seq_length):\n",
    "        _x = x[i:i + seq_length]\n",
    "        dataX.append(_x)\n",
    "    \n",
    "#     # Mix Data (to make it similar to i.i.d)\n",
    "#     idx = np.random.permutation(len(dataX))\n",
    "    \n",
    "#     outputX = []\n",
    "#     for i in range(len(dataX)):\n",
    "#         outputX.append(dataX[idx[i]])\n",
    "#     return outputX\n",
    "    return dataX\n",
    "    \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# n_time_series = 10\n",
    "# x_reduced = [str(i) for i in range(n_time_series)]\n",
    "# x_reduced\n",
    "\n",
    "# x_random = {}\n",
    "# for i in range(n_time_series):\n",
    "#     x_random[x_reduced[i]] = np.random.uniform(low=0, \n",
    "#                                                 high=1,\n",
    "#                                                 size=500)\n",
    "    \n",
    "# x_random_df = pd.DataFrame(data=x_random)\n",
    "\n",
    "\n",
    "# pld_complete_range = x_random_df\n",
    "# pld_complete_range\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "pld_complete_range = pd.read_csv(\"/home/soroosh/Desktop/SearchOX/data/pld_complete_range.csv\",\n",
    "                                 index_col=False)\n",
    "\n",
    "\n",
    "# pld_complete_range.head()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(370, 377, 378)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pos_idx = pld_complete_range.columns.get_loc(\"pos\")\n",
    "ic_idx = pld_complete_range.columns.get_loc(\"Ic\")\n",
    "ic_norm_idx = pld_complete_range.columns.get_loc(\"Ic_norm\")\n",
    "\n",
    "pos_idx, ic_idx, ic_norm_idx\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for i in range(379):\n",
    "#     print(pld_complete_range.columns[i], \"# unique values of :\"\n",
    "#           ,len(set(pld_complete_range.iloc[:, i].values)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "features_name = pld_complete_range.columns.to_list()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "without_stats_features = ['Speed', 'X FWHM', 'Y FWHM', 'R FWHM',\n",
    "                          'Coolness', 'Coolness_neg', 'Ic', 'Ic_norm']  # 'pos',\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "features_to_keep_names = []\n",
    "\n",
    "for feature in features_name:\n",
    "    if \"mean\" in feature:\n",
    "        features_to_keep_names.append(feature)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "for f in without_stats_features:\n",
    "    features_to_keep_names.append(f)\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "features_to_keep_indices = []\n",
    "\n",
    "for feature in features_to_keep_names:\n",
    "    features_to_keep_indices.append(features_name.index(feature))\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "105"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(features_to_keep_indices)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "pld_complete_range_non_stats = pld_complete_range.loc[:, without_stats_features]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pld_complete_range_non_stats\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open (\"/home/soroosh/Desktop/SearchOX/data/controllable_features.pickle\", \"rb\") as fp:\n",
    "    controllable_features = pickle.load(fp)\n",
    "    \n",
    "\n",
    "with open (\"/home/soroosh/Desktop/SearchOX/data/physics_related_features.pickle\", \"rb\") as fp:\n",
    "    physics_related_features = pickle.load(fp)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pld_complete_range_reduced_phy = pld_complete_range.loc[:, physics_related_features]\n",
    "# pld_complete_range_reduced_phy\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_results(dataX, dataX_hat, seq_len, parameters, n_time_series, name):\n",
    "\n",
    "    print(seq_len, \"\\n\", \n",
    "         parameters\n",
    "         )\n",
    "    \n",
    "    # save the results flatted along the windows_len\n",
    "    gen_data_arr = np.zeros([len(dataX_hat)*seq_len, n_time_series])\n",
    "    org_data_arr = np.zeros([len(dataX_hat)*seq_len, n_time_series])\n",
    "\n",
    "\n",
    "    interval = 0\n",
    "    for i in range(len(dataX_hat)):\n",
    "        for j in range(seq_len):\n",
    "            gen_data_arr[interval, :] = dataX_hat[i][j, :]\n",
    "            org_data_arr[interval, :] = dataX[i][j, :]\n",
    "            interval += 1    \n",
    "\n",
    "    interval = 0\n",
    "    for idx in range(len(dataX_hat)):\n",
    "        for j in range(seq_len): \n",
    "            if len(set(gen_data_arr[interval, :] == dataX_hat[idx][j, :])) != 1:\n",
    "                print(idx)\n",
    "            if len(set(org_data_arr[interval, :] == dataX[idx][j, :])) != 1:\n",
    "                print(idx)\n",
    "\n",
    "            interval += 1\n",
    "    \n",
    "     # flatted data for DTW\n",
    "    np.savetxt(\"gan_data/gen_data_gan-\"+\\\n",
    "               str(seq_len) + \"-\" + str(parameters[\"num_layers\"])+\\\n",
    "               \"-\" + str(parameters[\"iterations\"])+\\\n",
    "               name, \n",
    "          gen_data_arr)\n",
    "    \n",
    "        \n",
    "    np.savetxt(\"gan_data/org_data_gan-\"+\\\n",
    "               str(seq_len) + \"-\" + str(parameters[\"num_layers\"])+\\\n",
    "               \"-\" + str(parameters[\"iterations\"])+\\\n",
    "               name, \n",
    "              org_data_arr)\n",
    "    \n",
    "    with open (\"gan_data/dataX\" +\\\n",
    "               str(seq_len) + \"-\" +\\\n",
    "               str(parameters[\"num_layers\"])+\\\n",
    "               \"-\" + str(parameters[\"iterations\"])+\\\n",
    "               name, \"wb\") as fp:\n",
    "        pickle.dump(dataX, fp)\n",
    "        \n",
    "        \n",
    "    with open (\"gan_data/dataX_hat\" +\\\n",
    "               str(seq_len) + \"-\" +\\\n",
    "               str(parameters[\"num_layers\"])+\\\n",
    "               \"-\" + str(parameters[\"iterations\"])+\\\n",
    "               name, \"wb\") as fp:\n",
    "        pickle.dump(dataX_hat, fp)\n",
    "        \n",
    "    \n",
    "    return None\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def eval_scores(dataX, dataX_hat):\n",
    "    \n",
    "    # Output Initialization\n",
    "    Discriminative_Score = list()\n",
    "    Predictive_Score = list()\n",
    "\n",
    "    Acc = list()\n",
    "    Sub_Iteration = 2\n",
    "    for tt in range(Sub_Iteration):\n",
    "        Temp_Disc = discriminative_score_metrics(dataX, dataX_hat)\n",
    "        Acc.append(Temp_Disc)\n",
    "\n",
    "    Discriminative_Score.append(np.mean(Acc))\n",
    "\n",
    "    # 2. Predictive Performance\n",
    "    MAE_All = list()\n",
    "    for tt in range(Sub_Iteration):\n",
    "        MAE_All.append(predictive_score_metrics (dataX, dataX_hat))\n",
    "\n",
    "    Predictive_Score.append(np.mean(MAE_All))    \n",
    "    \n",
    "    # Print Results\n",
    "    print('Discriminative Score - Mean: ' + str(np.round(np.mean(Discriminative_Score), 4)) + ', Std: ' + str(np.round(np.std(Discriminative_Score),4)))\n",
    "    print('Predictive Score - Mean: ' + str(np.round(np.mean(Predictive_Score),4)) + ', Std: ' + str(np.round(np.std(Predictive_Score),4)))\n",
    "    \n",
    "    return Discriminative_Score, Predictive_Score\n",
    "\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_results(seq_len, parameters, name):\n",
    "    \n",
    "    print(\"Loading the saved results and data\")\n",
    "    \n",
    "    with open (\"gan_data/dataX\" +\\\n",
    "               str(seq_len) + \"-\" +\\\n",
    "               str(parameters[\"num_layers\"])+\\\n",
    "               \"-\" + str(parameters[\"iterations\"])+\\\n",
    "               name, \"rb\") as fp:\n",
    "        dataX=pickle.load(fp)\n",
    "    \n",
    "    \n",
    "    with open (\"gan_data/dataX_hat\" +\\\n",
    "               str(seq_len) + \"-\" +\\\n",
    "               str(parameters[\"num_layers\"])+\\\n",
    "               \"-\" + str(parameters[\"iterations\"])+\\\n",
    "               name, \"rb\") as fp:\n",
    "        dataX_hat=pickle.load(fp)\n",
    "        \n",
    "    \n",
    "    # flatted and saved data for DTW\n",
    "    org_data_gan = np.loadtxt(\"gan_data/org_data_gan-\"+\\\n",
    "                              str(seq_len) + \"-\" +\\\n",
    "                              str(parameters[\"num_layers\"])+\\\n",
    "                              \"-\" + str(parameters[\"iterations\"])+\\\n",
    "                              name\n",
    "                             )\n",
    "    \n",
    "    \n",
    "    gen_data_gan = np.loadtxt(\"gan_data/gen_data_gan-\"+\\\n",
    "                              str(seq_len) + \"-\" +\\\n",
    "                              str(parameters[\"num_layers\"])+\\\n",
    "                              \"-\" + str(parameters[\"iterations\"])+\\\n",
    "                              name\n",
    "                             )\n",
    "\n",
    "    \n",
    "    return dataX, dataX_hat, org_data_gan, gen_data_gan\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TSGAN (Time-Series)\n",
    "\n",
    "Paper: https://papers.nips.cc/paper/2019/file/c9efe5f26cd17ba6216bbe2a7d26d490-Paper.pdf\n",
    "\n",
    "Source code: https://github.com/vanderschaarlab/mlforhealthlabpub/tree/main/alg/timegan\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###### Define Model hyperparameters\n",
    "- Networks: Generator; Discriminator; Embedder; Recovery Network\n",
    "\n",
    "TimeGAN is a Generative model based on RNN networks. In this package the implemented version follows a very simple architecture that is shared by the four elements of the GAN.\n",
    "Similarly to other parameters, the architectures of each element should be optimized and tailored to the data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# project = \"SuperOX\"\n",
    "# data_type = \"reduced statistical features-dtw\"  # \"all statistical features\"\n",
    "\n",
    "\n",
    "\n",
    "#     run = init_a_wandb(name= \"GMM: \" + str(hyper_param),\n",
    "#                        project=project,\n",
    "#                        notes=\"Generating independent features - GMM \" + data_type, \n",
    "#                        group= \"GMM + CGAN - \" + data_type  # algorithm\n",
    "#                       )\n",
    "    \n",
    "\n",
    "\n",
    "#     run = wandb_metrics(run=run, y_trues=x_i_test, y_preds=x_i_hat)\n",
    "#     run = wandb_plot_predictions(run=run, algorithm= \"gmm: \" + str(hyper_param), \n",
    "#                                  y_trues=y_test, y_preds=y_i_hat.values.squeeze())\n",
    "    \n",
    "#     run = save_model(run=run, \n",
    "#                      model=gmm, \n",
    "#                      name=\"gmm\", \n",
    "#                      experiment_name=\"gmm 4 x_i -\" + data_type + \"-\" + str(hyper_param)\n",
    "#                     )  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# np.random.seed(42)\n",
    "# n_samples = pld_complete_range_reduced.shape[0]\n",
    "# n_train_samples = int(.7*n_samples)\n",
    "# all_indices = np.arange(n_samples).tolist()\n",
    "# train_indices = np.random.choice(all_indices, n_train_samples, replace=False).tolist()\n",
    "# test_indices = list(set(all_indices) - set(train_indices))\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train with different parameters:\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'non-stats-no-perem'"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sequence_lens = [3, 5, 10, 50] # 1 >> generate error in discriminative/predictive score\n",
    "number_layers = [3, 5]  \n",
    "\n",
    "iterations= 10000 \n",
    "name= \"non-stats-no-perem\"  # physics-perm, # physics-no-perm, \"non-stats-perem\", \n",
    "\n",
    "\n",
    "name\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_and_eval(data_to_use, \n",
    "                   sequence_lens,\n",
    "                   number_layers, \n",
    "                   name):\n",
    "    \n",
    "    for seq_len in sequence_lens:\n",
    "        for num_layers in number_layers:\n",
    "            print(\"seq_len:\", seq_len, \"num_layers:\", num_layers)\n",
    "            data_to_train = data_to_use.values\n",
    "            n_samples = data_to_train.shape[0]\n",
    "            n_time_series = data_to_train.shape[1]\n",
    "\n",
    "            dataX = prepare_data_for_tgan(x=data_to_train, seq_length=seq_len)\n",
    "\n",
    "            print(\"dataX[0].shape:\", dataX[0].shape)\n",
    "\n",
    "            #%% Newtork Parameters\n",
    "            parameters = dict()\n",
    "\n",
    "            parameters['hidden_dim'] = len(dataX[0][0,:]) * 4\n",
    "            parameters[\"num_layers\"] = num_layers\n",
    "            parameters['iterations'] = iterations\n",
    "            \n",
    "            parameters['batch_size'] = 128\n",
    "            parameters['module'] = 'gru'   # Other options: 'lstm' or 'lstmLN'\n",
    "\n",
    "            parameters['z_dim'] = len(dataX[0][0,:]) \n",
    "            \n",
    "            \n",
    "            \n",
    "            print('Parameters are ' + str(parameters))\n",
    "            print(\" \")\n",
    "\n",
    "            dataX_hat = timegan(dataX, parameters)   \n",
    "\n",
    "            print('Finish Synthetic Data Generation')\n",
    "\n",
    "            save_results(dataX=dataX, dataX_hat=dataX_hat, \n",
    "                         seq_len=seq_len,\n",
    "                         parameters=parameters, \n",
    "                         n_time_series=n_time_series,\n",
    "                         name=name\n",
    "                        )\n",
    "        \n",
    "    return None\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seq_len: 3 num_layers: 3\n",
      "dataX[0].shape: (3, 8)\n",
      "Parameters are {'hidden_dim': 32, 'num_layers': 3, 'iterations': 10000, 'batch_size': 128, 'module': 'gru', 'z_dim': 8}\n",
      " \n",
      "Start Embedding Network Training\n",
      "step: 0/10000, e_loss: 0.2663\n",
      "step: 1000/10000, e_loss: 0.0184\n",
      "step: 2000/10000, e_loss: 0.0184\n",
      "step: 3000/10000, e_loss: 0.0184\n",
      "step: 4000/10000, e_loss: 0.0134\n",
      "step: 5000/10000, e_loss: 0.0093\n",
      "step: 6000/10000, e_loss: 0.0071\n",
      "step: 7000/10000, e_loss: 0.0052\n",
      "step: 8000/10000, e_loss: 0.0046\n",
      "step: 9000/10000, e_loss: 0.0039\n",
      "Finish Embedding Network Training\n",
      "Start Training with Supervised Loss Only\n",
      "step: 0/10000, s_loss: 0.2243\n",
      "step: 1000/10000, s_loss: 0.0916\n",
      "step: 2000/10000, s_loss: 0.073\n",
      "step: 3000/10000, s_loss: 0.068\n",
      "step: 4000/10000, s_loss: 0.0639\n",
      "step: 5000/10000, s_loss: 0.0596\n",
      "step: 6000/10000, s_loss: 0.0553\n",
      "step: 7000/10000, s_loss: 0.0525\n",
      "step: 8000/10000, s_loss: 0.0493\n",
      "step: 9000/10000, s_loss: 0.0471\n",
      "Finish Training with Supervised Loss Only\n",
      "Start Joint Training\n",
      "step: 0/10000, d_loss: 2.0976, g_loss_u: 0.6754, g_loss_s: 0.1859, g_loss_v: 0.0956, e_loss_t0: 0.0332\n",
      "step: 1000/10000, d_loss: 1.7735, g_loss_u: 0.9892, g_loss_s: 0.048, g_loss_v: 0.0038, e_loss_t0: 0.0024\n",
      "step: 2000/10000, d_loss: 1.835, g_loss_u: 1.0742, g_loss_s: 0.0471, g_loss_v: 0.0025, e_loss_t0: 0.0021\n",
      "step: 3000/10000, d_loss: 1.6506, g_loss_u: 1.1313, g_loss_s: 0.0455, g_loss_v: 0.0033, e_loss_t0: 0.002\n",
      "step: 4000/10000, d_loss: 1.5292, g_loss_u: 1.0149, g_loss_s: 0.0468, g_loss_v: 0.0069, e_loss_t0: 0.0019\n"
     ]
    }
   ],
   "source": [
    "train_and_eval(data_to_use=pld_complete_range_non_stats, \n",
    "               sequence_lens=sequence_lens,\n",
    "               number_layers=number_layers,\n",
    "               name=name)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_and_eval(data_to_use=pld_complete_range_reduced_phy, \n",
    "#                sequence_lens=sequence_lens,\n",
    "#                number_layers=number_layers,\n",
    "#                name=\"physics\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate the obtained (and saved) results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"finish\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def eval_the_stored_results(name):\n",
    "    # name=\"\"  # \"\" for toy data, \"physics\", \"controlable\"\n",
    "    \n",
    "    sequence_lens = [3, 5, 10, 50] \n",
    "    number_layers = [3, 5,] \n",
    "    \n",
    "    discriminative_score = {}\n",
    "    predictive_score = {}\n",
    "    dtw_score = {}\n",
    "\n",
    "\n",
    "    for seq_len in sequence_lens:\n",
    "        for num_layers in number_layers:\n",
    "            print(\"seq_len:\", seq_len, \"num_layers:\", num_layers)\n",
    "\n",
    "            # data_to_use = pld_complete_range_reduced.values\n",
    "            # n_samples = data_to_use.shape[0]\n",
    "            # n_time_series = data_to_use.shape[1]\n",
    "\n",
    "\n",
    "            #%% Newtork Parameters\n",
    "            parameters = dict()\n",
    "\n",
    "            # parameters['hidden_dim'] = len(dataX[0][0,:]) * 4\n",
    "            parameters['num_layers'] = num_layers\n",
    "            parameters['iterations'] = iterations\n",
    "            parameters['batch_size'] = 128\n",
    "            parameters['module_name'] = 'gru'   # Other options: 'lstm' or 'lstmLN'\n",
    "            # parameters['z_dim'] = len(dataX[0][0,:]) \n",
    "\n",
    "            print('Parameters are ' + str(parameters))\n",
    "            print(\" \")\n",
    "\n",
    "            d, dh, org, gen = load_results(seq_len=seq_len, \n",
    "                                           parameters=parameters, \n",
    "                                           name=name)\n",
    "            \n",
    "            print(\"dataX[0].shape:\", d[0].shape, len(d), \"\\n\", \n",
    "                 \"dataX_hat[0].shape:\", dh[0].shape, len(dh)\n",
    "\n",
    "                 )\n",
    "            disc_scr, pred_scr = eval_scores(dataX=d, dataX_hat=dh)\n",
    "\n",
    "\n",
    "            dtw_tgan, _ = fastdtw(x=org, \n",
    "                                  y=gen,\n",
    "                                  dist=None)  # None then abs(x[i] - y[j]) will be used.\n",
    "\n",
    "            discriminative_score[str(seq_len)+\"-\"+str(num_layers)] = disc_scr\n",
    "\n",
    "            predictive_score[str(seq_len)+\"-\"+str(num_layers)] = pred_scr\n",
    "\n",
    "            dtw_score[str(seq_len)+\"-\"+str(num_layers)] = dtw_tgan\n",
    "\n",
    "\n",
    "\n",
    "    with open(\"gan_data/discriminative_score\" + name + \".pickle\", \"wb\") as fp:\n",
    "        pickle.dump(discriminative_score, fp)\n",
    "\n",
    "\n",
    "    with open(\"gan_data/predictive_score\" + name + \".pickle\", \"wb\") as fp:\n",
    "        pickle.dump(predictive_score, fp)\n",
    "\n",
    "\n",
    "    with open(\"gan_data/dtw_score\" + name + \".pickle\", \"wb\") as fp:\n",
    "        pickle.dump(dtw_score, fp)\n",
    "    \n",
    "    return discriminative_score, predictive_score, dtw_score\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "discriminative_score_toy, predictive_score_toy, dtw_score_toy = eval_the_stored_results(name=name)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "discriminative_score_phy, predictive_score_phy, dtw_score_phy = eval_the_stored_results(name=\"physics\")\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# with open(\"gan_data/discriminative_score.pickle\", \"rb\") as fp:\n",
    "#     ds = pickle.load(fp)\n",
    "    \n",
    "\n",
    "# with open(\"gan_data/predictive_score.pickle\", \"rb\") as fp:\n",
    "#     ps = pickle.load(fp)\n",
    "    \n",
    "    \n",
    "# with open(\"gan_data/dtw_score.pickle\", \"rb\") as fp:\n",
    "#     dtw = pickle.load(fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dataX_org_1half = prepare_data_for_tgan(x=data_to_train[:14000, ], seq_length=3)\n",
    "# dataX_org_2half = prepare_data_for_tgan(x=data_to_train[14000:, ], seq_length=3)\n",
    "\n",
    "# disc_scr_org_half, pred_scr_org_half = eval_scores(dataX=dataX_org_1half, dataX_hat=dataX_org_2half)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  Discriminative Score:\n",
    "\n",
    "For a quantitative measure of similarity, we train a post-hoc time-series classification model (by optimizing a 2-layer LSTM) to distinguish between sequences from the original and generated datasets. First, each original sequence is labeled real, and each generated sequence is labeled not real. Then, an off-the-shelf (RNN) classifier is trained to distinguish between the two classes as a standard supervised task. We then report the classification error on the held-out test set, which gives a quantitative assessment of (2).\n",
    "\n",
    "-  **(Lower the better)**\n",
    "\n",
    "##  Predictive Score:\n",
    "\n",
    "In order to be useful, the sampled data should inherit the predictive characteris- tics of the original. In particular, we expect TimeGAN to excel in capturing conditional distributions over time. Therefore, using the synthetic dataset, we train a post-hoc sequence-prediction model (by optimizing a 2-layer LSTM) to predict next-step temporal vectors over each input sequence. Then, we evaluate the trained model on the original dataset. Performance is measured in terms of the mean absolute error (MAE); for event-based data, the MAE is computed as |1− estimated probability that the event occurred|. This gives a quantitative assessment of (3).\n",
    "\n",
    "-  **Lower the better**\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"ON ORIGINALN DATA\"\n",
    "         \"\\n\", \n",
    "         \"    disc. score: %.4f\" % disc_scr_org[0],\n",
    "          \"\\t pred. score: %.4f\" %  pred_scr_ord[0],\n",
    "#           \"\\t dtw: %4f\" % dtw_score[k], \n",
    "         )\n",
    "\n",
    "for k, v in discriminative_score.items():\n",
    "    sl = k.split('-')[0]\n",
    "    nl = k.split('-')[1]\n",
    "    print(\"seq_len:\", sl, \"\\t num_layers:\", nl, \n",
    "         \"\\n\", \n",
    "         \"    disc. score: %.4f\" % v[0],\n",
    "          \"\\t pred. score: %.4f\" %  predictive_score[k][0],\n",
    "#           \"\\t dtw: %4f\" % dtw_score[k], \n",
    "         )\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- All of the obtained results are  close to the case when we apply the same framework on the original data.\n",
    "\n",
    "- Thus we can conclude that the generated data's quality is similar the original data\n",
    "\n",
    "- Please not that, in the experiment above our goal is not to obtain a low pred/disc score, but the goal is to obtain similar results when we usued the original data with the same prediction/discriminator algorithm."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dtw_org, _ = fastdtw(x=data_to_train, y=data_to_train, dist=None)  # None then abs(x[i] - y[j]) will be used.\n",
    "dtw_org\n",
    "\n",
    "dtw_org_half, _ = fastdtw(x=data_to_train[:5500, :],\n",
    "                          y=data_to_train[5500:, :], \n",
    "                          dist=None)  # None then abs(x[i] - y[j]) will be used.\n",
    "\n",
    "dtw_org_half\n",
    "\n",
    "\n",
    "# dtw, path = fastdtw(x, y, dist=euclidean)\n",
    "rnd_idx = np.random.choice(np.arange(0, n_samples), size=n_samples, replace=False)\n",
    "data_to_train_shuf = data_to_train[rnd_idx, :]\n",
    "dtw_shuf, _ = fastdtw(x=data_to_train, y=data_to_train_shuf, dist=None)  # None then abs(x[i] - y[j]) will be used.\n",
    "\n",
    "dtw_shuf\n",
    "\n",
    "x_random = {}\n",
    "for i in range(data_to_train.shape[1]):\n",
    "    x_random[i] = np.random.uniform(low=-1,  # x_reduced.iloc[:, i].min() \n",
    "                                    high=+1,  # x_reduced.iloc[:, i].max()\n",
    "                                    size=n_samples)\n",
    "    \n",
    "x_random_df = pd.DataFrame(data=x_random)\n",
    "\n",
    "\n",
    "dtw_rnd, _ = fastdtw(x=data_to_train, y=x_random_df, dist=None)  # None then abs(x[i] - y[j]) will be used.\n",
    "\n",
    "print(\" DTW of Original Data    : %.3f\" % dtw_org, \"\\n\", \n",
    "      \"DTW of Shuffled Data    : %.3f\" % dtw_shuf, \"\\n\",  \n",
    "      \"DTW of half Data        : %.3f\" % dtw_org_half, \"\\n\"\n",
    "      \" DTW of Random Data      : %.3f\" % dtw_rnd, \"\\n\", \n",
    "     )\n",
    "\n",
    "\n",
    "for k, v in discriminative_score.items():\n",
    "    sl = k.split('-')[0]\n",
    "    nl = k.split('-')[1]\n",
    "    print(\"seq_len:\", sl, \"\\t num_layers:\", nl, \n",
    "          \"\\t DTW: %.3f\" % dtw_score[k], \n",
    "         )\n",
    "    \n",
    "    \n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- The minimum DTW obtained when seq_len=3, num_layers=5\n",
    "- Comparing the DTW of the three first items with the Random Data, we can conclude that, even regarding this metrics also the quality is acceptable.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "for seq_len in sequence_lens:\n",
    "    for num_layers in number_layers:\n",
    "        \n",
    "        data_to_train = pld_complete_range_reduced.values\n",
    "        n_samples = data_to_train.shape[0]\n",
    "        n_time_series = data_to_train.shape[1]\n",
    "        \n",
    "\n",
    "        #%% Newtork Parameters\n",
    "        parameters = dict()\n",
    "\n",
    "        # parameters['hidden_dim'] = len(dataX[0][0,:]) * 4\n",
    "        parameters['num_layers'] = num_layers\n",
    "        parameters['iterations'] = iterations\n",
    "        parameters['batch_size'] = 128\n",
    "        parameters['module_name'] = 'gru'   # Other options: 'lstm' or 'lstmLN'\n",
    "        # parameters['z_dim'] = len(dataX[0][0,:]) \n",
    "\n",
    "#         print('Parameters are ' + str(parameters))\n",
    "#         print(\" \")\n",
    "        \n",
    "        d, dh, org, gen = load_results(seq_len=seq_len,\n",
    "                                       parameters=parameters, \n",
    "                                       name=name)\n",
    "        \n",
    "        print(\"seq_len:\", seq_len, \"\\t num_layers:\", num_layers)\n",
    "        \n",
    "        \n",
    "        \n",
    "#         print(\"dataX[0].shape:\", d[0].shape, len(d), \"\\n\", \n",
    "#              \"dataX_hat[0].shape:\", dh[0].shape, len(dh)\n",
    "#              )\n",
    "#         PCA_Analysis (d, dh)\n",
    "#         tSNE_Analysis (d, dh)\n",
    "        visualization(d, dh, 'pca')\n",
    "        visualization(d, dh, 'tsne')\n",
    "        \n",
    "        \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_the_data(org_data, gen_data, low=0, up=100):\n",
    "    n = org_data.shape[1]\n",
    "    # fig, axs = plt.subplots(n, 1, figsize=(n, 1))\n",
    "    for i in range(n):\n",
    "        name = pld_complete_range_reduced.columns[i]\n",
    "        plt.figure(figsize=(15, 8))\n",
    "        plt.plot(org_data[low:up, i], 'g-', alpha=.9) \n",
    "        plt.plot(gen_data[low:up, i], 'b--', alpha=.7) \n",
    "        plt.legend([\"Org\", \"TGAN\"])\n",
    "        plt.title(name)\n",
    "        plt.show()\n",
    "        plt.close()\n",
    "\n",
    "    return None\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "for seq_len in sequence_lens:\n",
    "    for num_layers in number_layers:\n",
    "        \n",
    "        data_to_train = pld_complete_range_reduced.values\n",
    "        n_samples = data_to_train.shape[0]\n",
    "        n_time_series = data_to_train.shape[1]\n",
    "        \n",
    "\n",
    "        #%% Newtork Parameters\n",
    "        parameters = dict()\n",
    "\n",
    "        # parameters['hidden_dim'] = len(dataX[0][0,:]) * 4\n",
    "        parameters['num_layers'] = num_layers\n",
    "        parameters['iterations'] = iterations\n",
    "        parameters['batch_size'] = 128\n",
    "        parameters['module_name'] = 'gru'   # Other options: 'lstm' or 'lstmLN'\n",
    "        # parameters['z_dim'] = len(dataX[0][0,:]) \n",
    "\n",
    "        \n",
    "        d, dh, org, gen = load_results(seq_len=seq_len, \n",
    "                                       parameters=parameters, \n",
    "                                       name=name)\n",
    "        \n",
    "        print(\"seq_len:\", seq_len, \n",
    "              \"num_layers:\", num_layers)\n",
    "\n",
    "\n",
    "        plot_the_data(org_data=org, \n",
    "              gen_data=gen, \n",
    "              low=4000, up=4100)\n",
    "        \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# seq_len = 3\n",
    "# data_to_train = pld_complete_range_reduced.values\n",
    "# n_samples = data_to_train.shape[0]\n",
    "# n_time_series = data_to_train.shape[1]\n",
    "\n",
    "# dataX = prepare_data_for_tgan(x=data_to_train, seq_length=seq_len)\n",
    "# dataX[0].shape\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#%% Newtork Parameters\n",
    "# parameters = dict()\n",
    "\n",
    "# parameters['hidden_dim'] = len(dataX[0][0,:]) * 4\n",
    "# parameters['num_layers'] = 3\n",
    "# parameters['iterations'] = 10000\n",
    "# parameters['batch_size'] = 128\n",
    "# parameters['module_name'] = 'gru'   # Other options: 'lstm' or 'lstmLN'\n",
    "# parameters['z_dim'] = len(dataX[0][0,:]) \n",
    "\n",
    "# print('Parameters are ' + str(parameters))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dataX_hat = tgan(dataX, parameters)   \n",
    "\n",
    "# print('Finish Synthetic Data Generation')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(dataX_hat), dataX_hat[0].shape\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_the_data(org_data=org_data_gan_1_5,\n",
    "              gen_data=gen_data_gan_1_5,\n",
    "             low=9000, up=9300)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_the_data(org_data=org_data_gan_10_3, \n",
    "              gen_data=gen_data_gan_10_3,\n",
    "              low=9000, up=9300\n",
    "             )\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_the_data(org_data=org_data_gan_10_5, \n",
    "              gen_data=gen_data_gan_10_5,\n",
    "             low=9000, up=9300)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_the_data(org_data=org_data_gan_50_3, \n",
    "              gen_data=gen_data_gan_50_3,\n",
    "             low=9000, up=9300)\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from sklearn.decomposition import PCA\n",
    "# from sklearn.manifold import TSNE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def compute_pca(x_to_pca, n_components=2):\n",
    "#     return PCA(n_components=n_components).fit_transform(x_to_pca)\n",
    "\n",
    "\n",
    "# def compute_tsne(x_to_tsne, n_components=2):\n",
    "#     return TSNE(n_components=n_components, n_iter=300).fit_transform(x_to_tsne)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# x_org_pca = compute_pca(org_data_arr)\n",
    "# x_tgan_pca = compute_pca(gen_data_arr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# plt.figure(figsize=(15, 8))\n",
    "# plt.scatter(x_org_pca[:, 0], x_org_pca[:, 1], alpha=.9, c=\"g\")\n",
    "# plt.scatter(x_tgan_pca[:, 0], x_tgan_pca[:, 1], alpha=.2, c=\"b\")\n",
    "\n",
    "# plt.legend([\"Org\", \"TGAN\"])\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# x_org_tsne = compute_tsne(org_data_arr)\n",
    "# x_tgan_tsne = compute_tsne(gen_data_arr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# plt.figure(figsize=(15, 8))\n",
    "# plt.scatter(x_org_tsne[:, 0], x_org_tsne[:, 1], alpha=.9, c=\"g\")\n",
    "# plt.scatter(x_tgan_tsne[:, 0], x_tgan_tsne[:, 1], alpha=.2, c=\"b\")\n",
    "\n",
    "# plt.legend([\"Org\", \"TGAN\"])\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tf1_venv_jun21",
   "language": "python",
   "name": "tf1_venv_jun21"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}