seq2seq-model-training-exporting.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# Time-series prediction using a seq2seq model\n",
    "\n",
    "### Version: v1.2\n",
    "#### Main features/components: grid search, seq2seq model (variable seq length and dimension), mock dataset generation, TF Model export. The input and output tensors (for TF Serving export) in this version are multidimensional.\n",
    "\n",
    "### Licenses/Credits\n",
    "* Copyright (c) 2019, PatternedScience Inc. This code was originally run on the [UniAnalytica platform](https://www.unianalytica.com/), is published by PatternedScience Inc. on GitHub and is licensed under the terms of Apache License 2.0; a copy of the license is available in the GitHub repository;\n",
    "\n",
    "* The seq2seq model, mock dataset generation, and the visualization code snippets are based on [the work by Guillaume Chevalier](https://github.com/guillaume-chevalier/seq2seq-signal-prediction) (MIT License).\n",
    "\n",
    "* Adding seq2seq decoder input feedback mechanism to the model through a modification based on code done by [Weimin Wang](https://github.com/aaxwaz/Multivariate-Time-Series-forecast-using-seq2seq-in-TensorFlow/blob/master/build_model_basic.py#L75)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "### Package imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "from os.path import isdir, join\n",
    "import sys\n",
    "from sys import exc_info\n",
    "import logging\n",
    "import shutil\n",
    "import time\n",
    "from datetime import datetime\n",
    "import pytz\n",
    "import random\n",
    "import math\n",
    "from operator import itemgetter\n",
    "import itertools\n",
    "import numpy as np\n",
    "print('Numpy version: ' + np.__version__)\n",
    "import pandas as pd\n",
    "print('Pandas version: ' + pd.__version__)\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "### Package imports (TensorFlow)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [],
   "source": [
    "logging.getLogger('tensorflow').disabled = True # Disabling TF logging\n",
    "import tensorflow as tf\n",
    "print('TensorFlow version: ' + tf.__version__)\n",
    "from tensorflow.core.framework import summary_pb2\n",
    "from tensorflow.python.client import device_lib\n",
    "# Letting the code run with more recent versions of TF:\n",
    "tf.nn.seq2seq = tf.contrib.legacy_seq2seq\n",
    "tf.nn.rnn_cell = tf.contrib.rnn\n",
    "tf.nn.rnn_cell.GRUCell = tf.contrib.rnn.GRUCell\n",
    "\n",
    "# 4 imports related to seq2seq feedback version:\n",
    "from tensorflow.python.ops import variable_scope\n",
    "from tensorflow.python.framework import dtypes\n",
    "from tensorflow.contrib import rnn\n",
    "import copy\n",
    "\n",
    "from tensorflow.python.client import device_lib\n",
    "def get_available_gpus():\n",
    "    local_device_protos = device_lib.list_local_devices()\n",
    "    return [x.physical_device_desc for x in local_device_protos if x.device_type == 'GPU']\n",
    "print('-------------------------')\n",
    "print('GPU info: ' + str(get_available_gpus()))\n",
    "print('-------------------------')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "### Fixed global params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [],
   "source": [
    "# Parameters that are commented are moved to the grid search parameters section\n",
    "params = {}\n",
    "params['parent_dir'] = '/workspace/notebooks/unianalytica/group/time-series-tf-serving-folder/time-series-tf-serving'\n",
    "params['experiments_results_dir'] = join(params['parent_dir'],'experiments_results')\n",
    "params['saved_models_dir'] = join(params['parent_dir'],'saved_models') # Saving the session for reload during testing\n",
    "params['exported_models_dir'] = join(params['parent_dir'],'exported_models')\n",
    "\n",
    "params['exported_model_version'] = 1 # Integer (for TensorFlow Serving use)\n",
    "\n",
    "params['is_plotting_predictions'] = True\n",
    "params['nb_predictions'] = 5\n",
    "\n",
    "params['is_plotting_losses'] = True\n",
    "\n",
    "params['seq_length_in'] = 10\n",
    "params['seq_length_out'] = 5\n",
    "\n",
    "params['input_dim'] = 1 # 1 or 2\n",
    "params['output_dim'] = 1 # 1 or 2\n",
    "\n",
    "params['dataset_size'] = 1000\n",
    "params['trainset_portion'] = 0.6\n",
    "params['validset_portion'] = 0.2\n",
    "\n",
    "params['batch_size'] = 32  # 5, 100\n",
    "params['num_epoch'] = 5\n",
    "\n",
    "params['lr_decay'] = 0.92  # (default: 0.9), 0.92\n",
    "params['momentum'] = 0.5  # (default: 0.0), 0.5\n",
    "params['lambda_l2_reg'] = 0.003  # 0.003\n",
    "params['init_learning_rate'] = 0.007"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "### GridSearch params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [],
   "source": [
    "#params['hidden_dim'] = 12  # Count of hidden neurons in the recurrent units. \n",
    "#params['layers_stacked_count'] = 2  # Number of stacked recurrent cells, on the neural depth axis. \n",
    "\n",
    "# params['seq2seq_type'] = 'orig_tf_seq2seq' # 'orig_tf_seq2seq', 'decoder_feedback', 'decoder_feedback_disabled'\n",
    "# decoder_feedback: the output of the decoder at time t is fed back and becomes the input of the decoder at time t+1\n",
    "# decoder_feedback_disabled: it is technically like orig_tf_seq2seq, but it uses the local functions defined here, instead of original TF functions\n",
    "\n",
    "grid_params = [\n",
    "   zip(['hidden_dim' for _ in range(2)],[12,24])\n",
    "   ,zip(['layers_stacked_count' for _ in range(2)],[1,2])\n",
    "   ,zip(['seq2seq_type' for _ in range(2)],['orig_tf_seq2seq', 'decoder_feedback'])\n",
    "]\n",
    "\n",
    "# Make sure the items under the above \"Fixed global params\" and \"GridSearch params\" are \n",
    "# all mentioned in the section \"Copying from params dict to local vars\" in runExperiment() function below"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "### Creating the directories that hold the results\n",
    "Creating these sub-directories under parent_dir: `experiments_results`, `saved_models`, `exported_models`\n",
    "\n",
    "#### If they exist already, THEY ARE DELETED ALONG WITH THEIR CONTENTS."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [],
   "source": [
    "if isdir(params['experiments_results_dir']):\n",
    "    try:\n",
    "        shutil.rmtree(params['experiments_results_dir'])\n",
    "    except:\n",
    "        print('Cound not delete the existing directory: ' + params['experiments_results_dir'])\n",
    "        e = exc_info()[0]\n",
    "        print(\"Error: %s\" % e)    \n",
    "try:\n",
    "    os.makedirs(params['experiments_results_dir'])\n",
    "except:\n",
    "    print('Cound not create directory: ' + params['experiments_results_dir'])\n",
    "    e = sys.exc_info()[0]\n",
    "    print(\"Error: %s\" % e)\n",
    "    \n",
    "if isdir(params['saved_models_dir']):\n",
    "    try:\n",
    "        shutil.rmtree(params['saved_models_dir'])\n",
    "    except:\n",
    "        print('Cound not delete the existing directory: ' + params['saved_models_dir'])\n",
    "        e = exc_info()[0]\n",
    "        print(\"Error: %s\" % e)    \n",
    "try:\n",
    "    os.makedirs(params['saved_models_dir'])\n",
    "except:\n",
    "    print('Cound not create directory: ' + params['saved_models_dir'])\n",
    "    e = sys.exc_info()[0]\n",
    "    print(\"Error: %s\" % e)\n",
    "    \n",
    "if isdir(params['exported_models_dir']):\n",
    "    try:\n",
    "        shutil.rmtree(params['exported_models_dir'])\n",
    "    except:\n",
    "        print('Cound not delete the existing directory: ' + params['exported_models_dir'])\n",
    "        e = exc_info()[0]\n",
    "        print(\"Error: %s\" % e)    \n",
    "try:\n",
    "    os.makedirs(params['exported_models_dir'])\n",
    "except:\n",
    "    print('Cound not create directory: ' + params['exported_models_dir'])\n",
    "    e = sys.exc_info()[0]\n",
    "    print(\"Error: %s\" % e)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "### Creating the dataset (and dividing it into 3 sets: training, validation, testing)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [],
   "source": [
    "now = datetime.now(pytz.timezone('US/Eastern'))\n",
    "seconds_since_epoch_overall_start = time.mktime(now.timetuple())\n",
    "    \n",
    "dataset_size = params['dataset_size']\n",
    "trainset_portion = params['trainset_portion']\n",
    "validset_portion = params['validset_portion']\n",
    "\n",
    "seq_length_in = params['seq_length_in']\n",
    "seq_length_out = params['seq_length_out']\n",
    "input_dim = params['input_dim']\n",
    "output_dim = params['output_dim']\n",
    "\n",
    "dataset_x = []\n",
    "dataset_y = []\n",
    "for _ in range(dataset_size):\n",
    "    rand = random.random() * 2 * math.pi\n",
    "\n",
    "    sig1 = np.sin(np.linspace(0.0 * math.pi + rand,\n",
    "                              3.0 * math.pi + rand, seq_length_in + seq_length_out))\n",
    "    sig2 = np.cos(np.linspace(0.0 * math.pi + rand,\n",
    "                              3.0 * math.pi + rand, seq_length_in + seq_length_out))\n",
    "        \n",
    "    x1 = sig1[:seq_length_in]\n",
    "    y1 = sig1[seq_length_in:]\n",
    "    \n",
    "    if input_dim == 1:\n",
    "        x_=np.tile(x1[:,None], input_dim)\n",
    "    \n",
    "    elif input_dim == 2:\n",
    "        x2 = sig2[:seq_length_in]\n",
    "        x_ = np.array([x1, x2])\n",
    "        x_= x_.T\n",
    "    \n",
    "    if output_dim == 1:\n",
    "        y_=np.tile(y1[:,None], output_dim)\n",
    "    \n",
    "    elif output_dim == 2:\n",
    "        y2 = sig2[seq_length_in:]\n",
    "        y_ = np.array([y1, y2])\n",
    "        y_= y_.T\n",
    "\n",
    "    dataset_x.append(x_)\n",
    "    dataset_y.append(y_)\n",
    "\n",
    "dataset_x = np.array(dataset_x)\n",
    "dataset_y = np.array(dataset_y)\n",
    "\n",
    "print('dataset_x.shape: {}'.format(dataset_x.shape))\n",
    "print('dataset_y.shape: {}'.format(dataset_y.shape))\n",
    "\n",
    "trainset_x = dataset_x[:int(trainset_portion*dataset_size)]\n",
    "trainset_y = dataset_y[:int(trainset_portion*dataset_size)]\n",
    "validset_x = dataset_x[int(trainset_portion*dataset_size):int((trainset_portion+validset_portion)*dataset_size)]\n",
    "validset_y = dataset_y[int(trainset_portion*dataset_size):int((trainset_portion+validset_portion)*dataset_size)]\n",
    "testset_x = dataset_x[int((trainset_portion+validset_portion)*dataset_size):]\n",
    "testset_y = dataset_y[int((trainset_portion+validset_portion)*dataset_size):]\n",
    "\n",
    "# X shape: (dataset_size, seq_length, input_dim)\n",
    "# Y shape: (dataset_size, seq_length, output_dim)\n",
    "\n",
    "dataset_dict = {\n",
    "    'trainset_x': trainset_x,\n",
    "    'trainset_y': trainset_y,\n",
    "    'validset_x': validset_x,\n",
    "    'validset_y': validset_y,\n",
    "    'testset_x': testset_x,\n",
    "    'testset_y': testset_y\n",
    "}\n",
    "\n",
    "del trainset_x\n",
    "del trainset_y\n",
    "del validset_x\n",
    "del validset_y\n",
    "del testset_x\n",
    "del testset_y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "### Seq2Seq model (quick) intro"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "<figure>\n",
    "    <img src=\"https://cdn-images-1.medium.com/max/1400/1*Ismhi-muID5ooWf3ZIQFFg.png\" />\n",
    "</figure>\n",
    "\n",
    "\n",
    "`<GO>`: This is the input to the first time step of the decoder to let the decoder know when to start generating output.\n",
    "    \n",
    "#### Image created by <a href=\"https://towardsdatascience.com/sequence-to-sequence-model-introduction-and-concepts-44d9b41cd42d\" target=\"_blank_\">Manish Chablani</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "### Adding the feedback mechanism to TF's seq2seq implementation\n",
    "\n",
    "Function overides so that the output of the decoder at time t is fed back and becomes the input of the decoder at time t+1. Modifications based on code done by [Weimin Wang](https://github.com/aaxwaz/Multivariate-Time-Series-forecast-using-seq2seq-in-TensorFlow/blob/master/build_model_basic.py#L75); however, unlike there, we do not use the actual outputs as decoder inputs during training (we use them only to calculate loss, i.e., after decoder outputs are reshaped). `seq2seq_type == decoder_feedback` in params should therefore provide the needed feedback mechanism."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [],
   "source": [
    "def _loop_function(prev, i, weights_out, biases_out):\n",
    "  \"\"\"\n",
    "  For _rnn_decoder. Transform prev from dimension [batch_size x hidden_dim] \n",
    "  to [batch_size x output_dim], which will be used as decoder input of \n",
    "  next time step\n",
    "  # i is an integer, the step number; not used for now\n",
    "  \"\"\"\n",
    "  return tf.matmul(prev, weights_out) + biases_out\n",
    "\n",
    "def _rnn_decoder(decoder_inputs,\n",
    "                initial_state,\n",
    "                cell,\n",
    "                loop_function=None,\n",
    "                scope=None,\n",
    "                weights_out=None,\n",
    "                biases_out=None):\n",
    "  \"\"\"RNN decoder for the sequence-to-sequence model.\n",
    "  Args:\n",
    "    decoder_inputs: A list of 2D Tensors [batch_size x input_size].\n",
    "    initial_state: 2D Tensor with shape [batch_size x cell.state_size].\n",
    "    cell: rnn_cell.RNNCell defining the cell function and size.\n",
    "    loop_function: If not None, this function will be applied to the i-th output\n",
    "      in order to generate the i+1-st input, and decoder_inputs will be ignored,\n",
    "      except for the first element (\"GO\" symbol). This can be used for decoding,\n",
    "      but also for training to emulate http://arxiv.org/abs/1506.03099.\n",
    "      Signature -- loop_function(prev, i) = next\n",
    "        * prev is a 2D Tensor of shape [batch_size x output_size],\n",
    "        * i is an integer, the step number (when advanced control is needed),\n",
    "        * next is a 2D Tensor of shape [batch_size x input_size].\n",
    "    scope: VariableScope for the created subgraph; defaults to \"rnn_decoder\".\n",
    "  Returns:\n",
    "    A tuple of the form (outputs, state), where:\n",
    "      outputs: A list of the same length as decoder_inputs of 2D Tensors with\n",
    "        shape [batch_size x output_size] containing generated outputs.\n",
    "      state: The state of each cell at the final time-step.\n",
    "        It is a 2D Tensor of shape [batch_size x cell.state_size].\n",
    "        (Note that in some cases, like basic RNN cell or GRU cell, outputs and\n",
    "         states can be the same. They are different for LSTM cells though.)\n",
    "  \"\"\"\n",
    "  with variable_scope.variable_scope(scope or \"rnn_decoder\"):\n",
    "    state = initial_state\n",
    "    outputs = []\n",
    "    prev = None\n",
    "    for i, inp in enumerate(decoder_inputs):\n",
    "      if loop_function is not None and prev is not None:\n",
    "        with variable_scope.variable_scope(\"loop_function\", reuse=True):\n",
    "          inp = loop_function(prev, i, weights_out, biases_out)\n",
    "      if i > 0:\n",
    "        variable_scope.get_variable_scope().reuse_variables()\n",
    "      output, state = cell(inp, state)\n",
    "      outputs.append(output)\n",
    "      if loop_function is not None:\n",
    "        prev = output\n",
    "  return outputs, state\n",
    " \n",
    "def _basic_rnn_seq2seq(encoder_inputs,\n",
    "                      decoder_inputs,\n",
    "                      cell,\n",
    "                      feed_previous,\n",
    "                      dtype,\n",
    "                      scope,\n",
    "                      weights_out,\n",
    "                      biases_out\n",
    "                      ):\n",
    "  \"\"\"Basic RNN sequence-to-sequence model.\n",
    "  This model first runs an RNN to encode encoder_inputs into a state vector,\n",
    "  then runs decoder, initialized with the last encoder state, on decoder_inputs.\n",
    "  Encoder and decoder use the same RNN cell type, but don't share parameters.\n",
    "  Args:\n",
    "    encoder_inputs: A list of 2D Tensors [batch_size x input_size].\n",
    "    decoder_inputs: A list of 2D Tensors [batch_size x input_size].\n",
    "    feed_previous: Boolean; if True, only the first of decoder_inputs will be\n",
    "      used (the \"GO\" symbol), all other inputs will be generated by the previous\n",
    "      decoder output using _loop_function below. If False, decoder_inputs are used\n",
    "      as given (the standard decoder case).\n",
    "    dtype: The dtype of the initial state of the RNN cell (default: tf.float32).\n",
    "    scope: VariableScope for the created subgraph; default: \"basic_rnn_seq2seq\".\n",
    "  Returns:\n",
    "    A tuple of the form (outputs, state), where:\n",
    "      outputs: A list of the same length as decoder_inputs of 2D Tensors with\n",
    "        shape [batch_size x output_size] containing the generated outputs.\n",
    "      state: The state of each decoder cell in the final time-step.\n",
    "        It is a 2D Tensor of shape [batch_size x cell.state_size].\n",
    "  \"\"\"\n",
    "  with variable_scope.variable_scope(scope or \"basic_rnn_seq2seq\"):\n",
    "    enc_cell = copy.deepcopy(cell)\n",
    "    _, enc_state = rnn.static_rnn(enc_cell, encoder_inputs, dtype=dtype)\n",
    "    if feed_previous:\n",
    "        return _rnn_decoder(decoder_inputs, enc_state, cell, _loop_function, scope, weights_out, biases_out)\n",
    "    else:\n",
    "        return _rnn_decoder(decoder_inputs, enc_state, cell)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "### Defining the process that happens per experiment (i.e. each item in the Grid Search)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [],
   "source": [
    "def runExperiment(experiment, params, dataset_dict):\n",
    "\n",
    "    experiment_label = ''\n",
    "    for grid_param in experiment:\n",
    "        params[grid_param[0]] = grid_param[1]\n",
    "        experiment_label = experiment_label + grid_param[0] + str(grid_param[1]) + '|'\n",
    "        \n",
    "    params['experiment_label'] = experiment_label\n",
    "    \n",
    "    # Copying from params dict to local vars\n",
    "    hidden_dim = params['hidden_dim']\n",
    "    layers_stacked_count = params['layers_stacked_count']\n",
    "    init_learning_rate = params['init_learning_rate']\n",
    "    num_epoch = params['num_epoch']\n",
    "    batch_size = params['batch_size']\n",
    "    is_plotting_losses = params['is_plotting_losses']\n",
    "    is_plotting_predictions = params['is_plotting_predictions']\n",
    "    nb_predictions = params['nb_predictions']\n",
    "    seq_length_in = params['seq_length_in']\n",
    "    seq_length_out = params['seq_length_out']\n",
    "    input_dim = params['input_dim']\n",
    "    output_dim = params['output_dim']\n",
    "    dataset_size = params['dataset_size']\n",
    "    trainset_portion = params['trainset_portion']\n",
    "    batch_size = params['batch_size']\n",
    "    momentum = params['momentum']\n",
    "    lr_decay = params['lr_decay']\n",
    "    lambda_l2_reg = params['lambda_l2_reg']    \n",
    "    seq2seq_type = params['seq2seq_type']\n",
    "\n",
    "    trainset_x = dataset_dict['trainset_x']\n",
    "    trainset_y = dataset_dict['trainset_y']\n",
    "    validset_x = dataset_dict['validset_x']\n",
    "    validset_y = dataset_dict['validset_y']\n",
    "    del dataset_dict\n",
    "    \n",
    "    res = {}\n",
    "    res['0_expId']= params['experiment_id']\n",
    "    res['1_TrLoss'] = 'NotSet'\n",
    "    res['2_VaLoss'] = 'NotSet'\n",
    "    res['3_Param#'] = 'NotSet' # Total number of model parameter\n",
    "    res['4_Exp.'] = experiment_label\n",
    "    res['5_Dur.'] = 'NotSet'\n",
    "\n",
    "    #############################################\n",
    "    now = datetime.now(pytz.timezone('US/Eastern'))\n",
    "    seconds_since_epoch_start = time.mktime(now.timetuple())\n",
    "    now_microsecond_start = now.microsecond\n",
    "    \n",
    "    # Designing the graph/model:\n",
    "    graph = tf.Graph()\n",
    "    with graph.as_default():\n",
    "        with tf.variable_scope('Seq2seq'):    \n",
    "\n",
    "            # Changed the list to Tensor so that it can be passed into the tf.saved_model.utils.build_tensor_info() function\n",
    "            enc_inp_tensor = tf.placeholder(tf.float32, shape=(seq_length_in, None, input_dim), name=\"enc_inp_tensor\")\n",
    "            enc_inp = [enc_inp_tensor[tf.constant(t),:,:] for t in range(seq_length_in)]\n",
    "\n",
    "            # Decoder: expected outputs (Changed list to Tensor, so that loss tensor can be called after model reload during testing)\n",
    "            expected_output_tensor = tf.placeholder(tf.float32, shape=(seq_length_out, None, output_dim), name=\"expected_output_tensor\")\n",
    "            \n",
    "            # Create a layers_stacked_count of stacked RNNs (GRU cells here). \n",
    "            cells = []\n",
    "            for i in range(layers_stacked_count):\n",
    "                with tf.variable_scope('RNN_{}'.format(i)):\n",
    "                    cells.append(tf.nn.rnn_cell.GRUCell(hidden_dim))\n",
    "                    # cells.append(tf.nn.rnn_cell.BasicLSTMCell(...))\n",
    "            cell = tf.nn.rnn_cell.MultiRNNCell(cells)\n",
    "            \n",
    "            dec_inp = [ tf.zeros_like(enc_inp[0], dtype=np.float32, name=\"GO\") ] + enc_inp[:seq_length_out-1]\n",
    "            \n",
    "            # For reshaping the output of the seq2seq RNN: \n",
    "            weights_out = tf.Variable(tf.random_normal([hidden_dim, output_dim]))\n",
    "            biases_out = tf.Variable(tf.random_normal([output_dim]), name=\"Bias_output\")\n",
    "            \n",
    "            if seq2seq_type == 'orig_tf_seq2seq':\n",
    "                # Here, the encoder and the decoder uses the same cell, HOWEVER,\n",
    "                # the weights aren't shared among the encoder and decoder, we have two\n",
    "                # sets of weights created under the hood according to that function's def. \n",
    "                dec_outputs, dec_memory = tf.nn.seq2seq.basic_rnn_seq2seq(\n",
    "                    enc_inp, \n",
    "                    dec_inp, \n",
    "                    cell\n",
    "                )\n",
    "            elif seq2seq_type == 'decoder_feedback':\n",
    "                dec_outputs, dec_memory = _basic_rnn_seq2seq(\n",
    "                    enc_inp,\n",
    "                    dec_inp,\n",
    "                    cell,\n",
    "                    feed_previous = True,\n",
    "                    dtype=dtypes.float32,\n",
    "                    scope='Seq2seq',\n",
    "                    weights_out=weights_out,\n",
    "                    biases_out=biases_out\n",
    "                )                \n",
    "            elif seq2seq_type == 'decoder_feedback_disabled':\n",
    "                dec_outputs, dec_memory = _basic_rnn_seq2seq(\n",
    "                    enc_inp,\n",
    "                    dec_inp,\n",
    "                    cell,\n",
    "                    feed_previous = False,\n",
    "                    dtype=dtypes.float32,\n",
    "                    scope='Seq2seq',\n",
    "                    weights_out=weights_out,\n",
    "                    biases_out=biases_out\n",
    "                )                \n",
    "\n",
    "            # Final outputs: with linear rescaling for enabling possibly large and unrestricted output values.\n",
    "            output_scale_factor = tf.Variable(1.0, name=\"Output_ScaleFactor\")\n",
    "\n",
    "            reshaped_outputs = [output_scale_factor*(tf.matmul(i, weights_out) + biases_out) for i in dec_outputs]\n",
    "            # Created this tensor so that it can be passed into the tf.saved_model.utils.build_tensor_info() function\n",
    "            reshaped_outputs_tensor = tf.stack(reshaped_outputs, 0, name=\"reshaped_outputs_tensor\")\n",
    "\n",
    "        # Training loss and optimizer\n",
    "        with tf.variable_scope('Loss'):\n",
    "            # L2 loss\n",
    "            output_loss = tf.reduce_mean(tf.nn.l2_loss(reshaped_outputs_tensor - expected_output_tensor))\n",
    "\n",
    "            # L2 regularization\n",
    "            reg_loss = 0\n",
    "            for tf_var in tf.trainable_variables():\n",
    "                if not (\"Bias\" in tf_var.name or \"Output_\" in tf_var.name):\n",
    "                    reg_loss += tf.reduce_mean(tf.nn.l2_loss(tf_var))\n",
    "\n",
    "            loss = tf.identity(output_loss + lambda_l2_reg * reg_loss, name=\"loss\")\n",
    "\n",
    "        with tf.variable_scope('Optimizer'):\n",
    "            optimizer = tf.train.RMSPropOptimizer(init_learning_rate, decay=lr_decay, momentum=momentum)\n",
    "            train_op = optimizer.minimize(loss)\n",
    "\n",
    "        # Calculate and print the total number of model parameters:\n",
    "        total_parameters = 0\n",
    "        for variable in tf.trainable_variables():\n",
    "            shape = variable.get_shape()\n",
    "            variable_parametes = 1\n",
    "            for dim in shape:\n",
    "                variable_parametes *= dim.value\n",
    "            total_parameters += variable_parametes\n",
    "        print('Total number of model parameter: ' + \\\n",
    "            str( np.around( total_parameters/1000.0 , decimals=3) ) + \\\n",
    "            ' K (' + str(total_parameters) + ')')\n",
    "        # Ref: http://stackoverflow.com/a/38161314\n",
    "        res['3_Param#'] = str(np.around( total_parameters/1000.0 , decimals=1)) + 'K'\n",
    "\n",
    "        # Training\n",
    "        train_losses = []\n",
    "        valid_losses = []\n",
    "\n",
    "        init = tf.global_variables_initializer()\n",
    "        saver = tf.train.Saver()\n",
    "        \n",
    "    # Launch the graph\n",
    "    with tf.Session(graph=graph) as sess:\n",
    "        # Run all the initializers to prepare the trainable parameters.\n",
    "        sess.run(init)\n",
    "\n",
    "        step = 1\n",
    "        # Keep training until reaching the max iterations\n",
    "        while step * batch_size <= num_epoch*len(trainset_x) :\n",
    "            offset = (step * batch_size) % (len(trainset_x) - batch_size)\n",
    "            X = trainset_x[offset:(offset + batch_size)].transpose((1, 0, 2)) # reshaped to (seq_length_in, batch_size, input_dim)\n",
    "            Y = trainset_y[offset:(offset + batch_size)].transpose((1, 0, 2))\n",
    "            feed_dict = {enc_inp_tensor: X}\n",
    "            feed_dict.update({expected_output_tensor: Y})\n",
    "            _, train_loss = sess.run([train_op, loss], feed_dict)\n",
    "            train_losses.append(train_loss)\n",
    "            \n",
    "            if step % 10 == 0:\n",
    "                # Test on validation set\n",
    "                n_batch_valid = len(validset_x) // batch_size\n",
    "                validset_len = n_batch_valid * batch_size\n",
    "                batch_valid_losses = []        \n",
    "                valid_predictions = np.zeros(shape=(validset_len, output_dim, seq_length_out)) # (batch, dimension/KPIs, time)\n",
    "                for i in range(n_batch_valid):\n",
    "                    #print('Predictions done for batch #{} of {} [validation set]'.format(i+1,n_batch_valid))\n",
    "                    valid_x = validset_x[i * batch_size: i * batch_size + batch_size].transpose((1, 0, 2)) # reshaped to (seq_length_in, batch_size, input_dim)\n",
    "                    valid_y = validset_y[i * batch_size: i * batch_size + batch_size].transpose((1, 0, 2))\n",
    "                    feed_dict = {enc_inp_tensor: valid_x}\n",
    "                    feed_dict.update({expected_output_tensor: valid_y})\n",
    "                    batch_valid_predictions, batch_valid_loss = sess.run([reshaped_outputs_tensor, loss], feed_dict)\n",
    "                    batch_valid_losses.append(batch_valid_loss)\n",
    "                    valid_predictions[i * batch_size: i * batch_size + batch_size] = batch_valid_predictions.transpose((1, 2, 0)) # reshaped to (batch_size, output_dim, seq_length_out)\n",
    "                #print('Predictions on validset done. Shape of predictions array: {}'.format(valid_predictions.shape))\n",
    "                # valid_predictions is currently not being used, but generated anyway in case we want to look at individual predictions\n",
    "\n",
    "                valid_loss = np.mean(batch_valid_losses)\n",
    "                valid_losses.append(valid_loss)\n",
    "                #print(\"Valid loss: {}\".format(valid_loss))\n",
    "        \n",
    "                print(\"Step {}/{}, Train loss: {}, \\tValid loss: {}\".format(step, int(num_epoch*len(trainset_x)/batch_size), train_loss, valid_loss))\n",
    "\n",
    "            step += 1\n",
    "\n",
    "        mean_train_loss = np.mean(train_losses)\n",
    "        mean_valid_loss = np.mean(valid_losses)\n",
    "        print(\"End: mean train loss: {}, \\tmean valid loss: {}\".format(mean_train_loss, mean_valid_loss))\n",
    "        res['1_TrLoss'] = mean_train_loss\n",
    "        res['2_VaLoss'] = mean_valid_loss    \n",
    "\n",
    "        if is_plotting_losses:\n",
    "            # Plot loss over time:\n",
    "            plt.figure(figsize=(12, 6))\n",
    "            plt.plot(\n",
    "                np.array(range(0, len(valid_losses)))/float(len(valid_losses)-1)*(len(train_losses)-1), \n",
    "                np.log(valid_losses), \n",
    "                label=\"Valid loss\"\n",
    "            )\n",
    "            plt.plot(\n",
    "                np.log(train_losses), \n",
    "                label=\"Train loss\"\n",
    "            )\n",
    "            plt.title(\"Training errors over time (on a logarithmic scale)\")\n",
    "            plt.xlabel('Iteration')\n",
    "            plt.ylabel('log(Loss)')\n",
    "            plt.legend(loc='best')\n",
    "            plt.show()\n",
    "\n",
    "        if is_plotting_predictions:\n",
    "            print(\"Visualizing {} predictions:\".format(nb_predictions))\n",
    "\n",
    "            indices = np.random.choice(len(validset_x), size= nb_predictions, replace=False)\n",
    "\n",
    "            X = np.array(itemgetter(*indices)(validset_x)).transpose((1, 0, 2))\n",
    "            Y = np.array(itemgetter(*indices)(validset_y)).transpose((1, 0, 2))\n",
    "\n",
    "            feed_dict = {enc_inp_tensor: X}\n",
    "            outputs = np.array(sess.run([reshaped_outputs_tensor], feed_dict)[0])\n",
    "\n",
    "            for j in range(nb_predictions): \n",
    "                plt.figure(figsize=(12, 3))\n",
    "\n",
    "                for k in range(input_dim):\n",
    "                    past = X[:,j,k]\n",
    "                    label1 = \"Seen (past) values\" if k==0 else \"_nolegend_\"\n",
    "                    plt.plot(range(len(past)), past, \"o--b\", label=label1)\n",
    "\n",
    "                for k in range(output_dim):\n",
    "                    expected = Y[:,j,k]\n",
    "                    pred = outputs[:,j,k]\n",
    "\n",
    "                    label2 = \"True future values\" if k==0 else \"_nolegend_\"\n",
    "                    label3 = \"Predictions\" if k==0 else \"_nolegend_\"\n",
    "                    plt.plot(range(len(past), len(expected)+len(past)), expected, \"x--b\", label=label2)\n",
    "                    plt.plot(range(len(past), len(pred)+len(past)), pred, \"o--y\", label=label3)\n",
    "\n",
    "                plt.legend(loc='best')\n",
    "                plt.title(\"Predictions v.s. true values\")\n",
    "                plt.show()    \n",
    "    \n",
    "        now = datetime.now(pytz.timezone('US/Eastern'))\n",
    "        seconds_since_epoch_end = time.mktime(now.timetuple())\n",
    "        now_microsecond_end = now.microsecond\n",
    "\n",
    "        res['5_Dur.'] = np.around( (seconds_since_epoch_end - seconds_since_epoch_start)/60.0 , decimals=2)\n",
    "        if seconds_since_epoch_end != seconds_since_epoch_start:\n",
    "            duration_str = 'Experiment processing took ' + \\\n",
    "            str(res['5_Dur.']) + ' minutes'\n",
    "            print(duration_str)\n",
    "        else:\n",
    "            duration_microseconds_part = now_microsecond_end - now_microsecond_start\n",
    "            duration_str = 'Experiment processing took ' + \\\n",
    "            str(res['5_Dur.']) + ' minutes and ' + str(duration_microseconds_part/1000) + ' milliseconds.'\n",
    "            print(duration_str)    \n",
    "\n",
    "\n",
    "        model_save_path = join(params['saved_models_dir'],'exp'+str(params['experiment_id']),'saved_model')\n",
    "        saver.save(sess, model_save_path)\n",
    "        print('Model was saved to ' + model_save_path)\n",
    "            \n",
    "        model_export_path = join(params['exported_models_dir'],'exp'+str(params['experiment_id']),str(params['exported_model_version']))\n",
    "        builder = tf.saved_model.builder.SavedModelBuilder(model_export_path)\n",
    "\n",
    "        # enc_inp_tensor.shape:  (seq_length_in, batch_size, input_dim) and with float32 as element type\n",
    "        # reshaped_outputs_tensor.shape: (seq_length_out, batch_size, output_dim) and with float32 as element type\n",
    "        tensor_info_x = tf.saved_model.utils.build_tensor_info(enc_inp_tensor)\n",
    "        tensor_info_y = tf.saved_model.utils.build_tensor_info(reshaped_outputs_tensor)\n",
    "\n",
    "        prediction_signature = (\n",
    "          tf.saved_model.signature_def_utils.build_signature_def(\n",
    "              inputs={'input_series': tensor_info_x},\n",
    "              outputs={'output_series': tensor_info_y},\n",
    "              method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME))\n",
    "\n",
    "        builder.add_meta_graph_and_variables(\n",
    "          sess, [tf.saved_model.tag_constants.SERVING],\n",
    "          signature_def_map={\n",
    "              tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:prediction_signature\n",
    "          })\n",
    "\n",
    "        builder.save()\n",
    "        print('Model was exported to ' + model_export_path)\n",
    "\n",
    "    return res"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "### Running the scenarios/experiments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [],
   "source": [
    "all_experiments = []\n",
    "for element in itertools.product(*grid_params):\n",
    "    all_experiments.append(element)\n",
    "\n",
    "print('all_experiments {}:'.format(len(all_experiments)))\n",
    "for x in all_experiments:\n",
    "    print(str(x))\n",
    "\n",
    "results = []\n",
    "i = 1\n",
    "for experiment in all_experiments:\n",
    "    print('')\n",
    "    print('--------------------------------------------------')\n",
    "    print('--------------------------------------------------')\n",
    "    print('Running Experiment ' + str(i) + ' of ' + str(len(all_experiments)))\n",
    "    params['experiment_id'] = i\n",
    "    print('Experiment: \"' + str(experiment))\n",
    "    print('Started at ' + str(datetime.now(pytz.timezone('US/Eastern'))))\n",
    "    res = runExperiment(experiment, params, dataset_dict)\n",
    "    print('Experiment \"' + str(experiment))\n",
    "    print('finished at '  + str(datetime.now(pytz.timezone('US/Eastern'))))\n",
    "    print('with the result: ' + str(res))\n",
    "    results.append(res)\n",
    "    resultsDF = pd.DataFrame(results)\n",
    "    resultsDF.to_csv(join(params['experiments_results_dir'],'experiments_resultsDF.csv'), sep=',', encoding='utf-8')\n",
    "    i += 1\n",
    "\n",
    "print('-------------------------------')\n",
    "print('-------------------------------')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "### Best experiment/model according to validation set results is determined"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [],
   "source": [
    "#resultsDF = pd.read_csv(join(params['experiments_results_dir'],'experiments_resultsDF.csv'), sep=',', index_col=0, header=0)\n",
    "resultsDF = resultsDF.sort_values(['2_VaLoss'], ascending=[1])\n",
    "resultsDF_first_choice_ix = resultsDF.index.tolist()[0]\n",
    "chosen_exp_id = resultsDF.at[resultsDF_first_choice_ix, '0_expId']\n",
    "print('Experiment with ID of {} has the lowest validation loss during training.'.format(chosen_exp_id))\n",
    "\n",
    "print('All results:')\n",
    "print(resultsDF)\n",
    "\n",
    "now = datetime.now(pytz.timezone('US/Eastern'))\n",
    "seconds_since_epoch_overall_end = time.mktime(now.timetuple())\n",
    "\n",
    "print()\n",
    "print('Processing all experiments took ' + str( np.around( (seconds_since_epoch_overall_end - seconds_since_epoch_overall_start)/60.0 , decimals=2) ) + ' minutes')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "### Picking the best model based on validation set results and reporting its result on the test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [],
   "source": [
    "chosen_exp_id = None # If None, the experiment having the lowest loss on validation set is chosen\n",
    "\n",
    "batch_size = params['batch_size']\n",
    "is_plotting_predictions = params['is_plotting_predictions']\n",
    "nb_predictions = params['nb_predictions']\n",
    "output_dim = params['output_dim']\n",
    "seq_length_out = params['seq_length_out']\n",
    "\n",
    "testset_x = dataset_dict['testset_x']\n",
    "testset_y = dataset_dict['testset_y']\n",
    "\n",
    "n_batch_test = len(testset_x) // batch_size\n",
    "testset_len = n_batch_test * batch_size\n",
    "\n",
    "if chosen_exp_id == None:\n",
    "    resultsDF = pd.read_csv(join(params['experiments_results_dir'],'experiments_resultsDF.csv'), sep=',', index_col=0, header=0)\n",
    "    resultsDF = resultsDF.sort_values(['2_VaLoss'], ascending=[1])\n",
    "    resultsDF_first_choice_ix = resultsDF.index.tolist()[0]\n",
    "    chosen_exp_id = resultsDF.at[resultsDF_first_choice_ix, '0_expId']\n",
    "\n",
    "print(\"Testing model with exp_id {}\".format(chosen_exp_id))\n",
    "\n",
    "print('Experiment label: ', resultsDF.at[resultsDF_first_choice_ix, '4_Exp.'])\n",
    "\n",
    "model_save_path = join(params['saved_models_dir'],'exp'+str(chosen_exp_id),'saved_model')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "### Loading the saved model, predicting on testset, and plotting some predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [],
   "source": [
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "    with tf.Session() as sess:\n",
    "        saver = tf.train.import_meta_graph(model_save_path+'.meta')\n",
    "        saver.restore(sess, model_save_path)\n",
    "        graph = tf.get_default_graph()\n",
    "        enc_inp_tensor = graph.get_tensor_by_name(\"Seq2seq/enc_inp_tensor:0\")\n",
    "        expected_output_tensor = graph.get_tensor_by_name(\"Seq2seq/expected_output_tensor:0\")\n",
    "        reshaped_outputs_tensor = graph.get_tensor_by_name(\"Seq2seq/reshaped_outputs_tensor:0\")\n",
    "        loss_tensor = graph.get_tensor_by_name(\"Loss/loss:0\")\n",
    "\n",
    "        print('Model successfully restored')\n",
    "        \n",
    "        test_losses = []        \n",
    "        predictions = np.zeros(shape=(testset_len, output_dim, seq_length_out)) # (batch, dimension/KPIs, time)\n",
    "        for i in range(n_batch_test):\n",
    "            print('Predictions done for batch #{} of {}'.format(i+1,n_batch_test))\n",
    "            test_x = testset_x[i * batch_size: i * batch_size + batch_size].transpose((1, 0, 2)) # reshaped to (seq_length_in, batch_size, input_dim)\n",
    "            test_y = testset_y[i * batch_size: i * batch_size + batch_size].transpose((1, 0, 2))\n",
    "            feed_dict = {enc_inp_tensor: test_x}\n",
    "            feed_dict.update({expected_output_tensor: test_y})\n",
    "            predictions_batch, test_loss = sess.run([reshaped_outputs_tensor, loss_tensor], feed_dict)\n",
    "            test_losses.append(test_loss)\n",
    "            predictions[i * batch_size: i * batch_size + batch_size] = predictions_batch.transpose((1, 2, 0)) # reshaped to (batch_size, output_dim, seq_length_out)\n",
    "        print('Predictions on testset done. Shape of predictions array: {}'.format(predictions.shape))\n",
    "\n",
    "        mean_test_loss = np.mean(test_losses)\n",
    "        print(\"Test loss: {}\".format(mean_test_loss))\n",
    "        \n",
    "        if is_plotting_predictions:\n",
    "            print(\"Visualizing {} predictions:\".format(nb_predictions))\n",
    "\n",
    "            indices = np.random.choice(len(predictions), size= nb_predictions, replace=False)\n",
    "\n",
    "            X = np.array(itemgetter(*indices)(testset_x)).transpose((1, 0, 2))\n",
    "            Y = np.array(itemgetter(*indices)(testset_y)).transpose((1, 0, 2))\n",
    "            outputs = np.array(itemgetter(*indices)(predictions)).transpose((2, 0, 1))\n",
    "\n",
    "            for j in range(nb_predictions): \n",
    "                plt.figure(figsize=(12, 3))\n",
    "\n",
    "                for k in range(input_dim):\n",
    "                    past = X[:,j,k]\n",
    "                    label1 = \"Seen (past) values\" if k==0 else \"_nolegend_\"\n",
    "                    plt.plot(range(len(past)), past, \"o--b\", label=label1)\n",
    "\n",
    "                for k in range(output_dim):\n",
    "                    expected = Y[:,j,k]\n",
    "                    pred = outputs[:,j,k]\n",
    "\n",
    "                    label2 = \"True future values\" if k==0 else \"_nolegend_\"\n",
    "                    label3 = \"Predictions\" if k==0 else \"_nolegend_\"\n",
    "                    plt.plot(range(len(past), len(expected)+len(past)), expected, \"x--b\", label=label2)\n",
    "                    plt.plot(range(len(past), len(pred)+len(past)), pred, \"o--y\", label=label3)\n",
    "\n",
    "                plt.legend(loc='best')\n",
    "                plt.title(\"Predictions v.s. true values\")\n",
    "                plt.show()"
   ]
  }
 ],
 "metadata": {
  "celltoolbar": "Slideshow",
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}