add in real case study for Beijing pollution data set

build_model_basic.py

@@ -0,0 +1,200 @@
+import tensorflow as tf
+from tensorflow.contrib import rnn
+from tensorflow.python.ops import variable_scope
+from tensorflow.python.framework import dtypes
+import copy
+
+## Parameters
+learning_rate = 0.01
+lambda_l2_reg = 0.003  
+
+## Network Parameters
+# length of input signals
+input_seq_len = 15 
+# length of output signals
+output_seq_len = 20 
+# size of LSTM Cell
+hidden_dim = 64 
+# num of input signals
+input_dim = 1
+# num of output signals
+output_dim = 1
+# num of stacked lstm layers 
+num_stacked_layers = 2 
+# gradient clipping - to avoid gradient exploding
+GRADIENT_CLIPPING = 2.5 
+
+def build_graph(feed_previous = False):
+
+    tf.reset_default_graph()
+
+    global_step = tf.Variable(
+                  initial_value=0,
+                  name="global_step",
+                  trainable=False,
+                  collections=[tf.GraphKeys.GLOBAL_STEP, tf.GraphKeys.GLOBAL_VARIABLES])
+
+    weights = {
+        'out': tf.get_variable('Weights_out', \
+                               shape = [hidden_dim, output_dim], \
+                               dtype = tf.float32, \
+                               initializer = tf.truncated_normal_initializer()),
+    }
+    biases = {
+        'out': tf.get_variable('Biases_out', \
+                               shape = [output_dim], \
+                               dtype = tf.float32, \
+                               initializer = tf.constant_initializer(0.)),
+    }
+
+    with tf.variable_scope('Seq2seq'):
+        # Encoder: inputs
+        enc_inp = [
+            tf.placeholder(tf.float32, shape=(None, input_dim), name="inp_{}".format(t))
+               for t in range(input_seq_len)
+        ]
+
+        # Decoder: target outputs
+        target_seq = [
+            tf.placeholder(tf.float32, shape=(None, output_dim), name="y".format(t))
+              for t in range(output_seq_len)
+        ]
+
+        # Give a "GO" token to the decoder. 
+        # If dec_inp are fed into decoder as inputs, this is 'guided' training; otherwise only the 
+        # first element will be fed as decoder input which is then 'un-guided'
+        dec_inp = [ tf.zeros_like(target_seq[0], dtype=tf.float32, name="GO") ] + target_seq[:-1]
+
+        with tf.variable_scope('LSTMCell'): 
+            cells = []
+            for i in range(num_stacked_layers):
+                with tf.variable_scope('RNN_{}'.format(i)):
+                    cells.append(tf.contrib.rnn.LSTMCell(hidden_dim))
+            cell = tf.contrib.rnn.MultiRNNCell(cells)
+
+        def _rnn_decoder(decoder_inputs,
+                        initial_state,
+                        cell,
+                        loop_function=None,
+                        scope=None):
+          """RNN decoder for the sequence-to-sequence model.
+          Args:
+            decoder_inputs: A list of 2D Tensors [batch_size x input_size].
+            initial_state: 2D Tensor with shape [batch_size x cell.state_size].
+            cell: rnn_cell.RNNCell defining the cell function and size.
+            loop_function: If not None, this function will be applied to the i-th output
+              in order to generate the i+1-st input, and decoder_inputs will be ignored,
+              except for the first element ("GO" symbol). This can be used for decoding,
+              but also for training to emulate http://arxiv.org/abs/1506.03099.
+              Signature -- loop_function(prev, i) = next
+                * prev is a 2D Tensor of shape [batch_size x output_size],
+                * i is an integer, the step number (when advanced control is needed),
+                * next is a 2D Tensor of shape [batch_size x input_size].
+            scope: VariableScope for the created subgraph; defaults to "rnn_decoder".
+          Returns:
+            A tuple of the form (outputs, state), where:
+              outputs: A list of the same length as decoder_inputs of 2D Tensors with
+                shape [batch_size x output_size] containing generated outputs.
+              state: The state of each cell at the final time-step.
+                It is a 2D Tensor of shape [batch_size x cell.state_size].
+                (Note that in some cases, like basic RNN cell or GRU cell, outputs and
+                 states can be the same. They are different for LSTM cells though.)
+          """
+          with variable_scope.variable_scope(scope or "rnn_decoder"):
+            state = initial_state
+            outputs = []
+            prev = None
+            for i, inp in enumerate(decoder_inputs):
+              if loop_function is not None and prev is not None:
+                with variable_scope.variable_scope("loop_function", reuse=True):
+                  inp = loop_function(prev, i)
+              if i > 0:
+                variable_scope.get_variable_scope().reuse_variables()
+              output, state = cell(inp, state)
+              outputs.append(output)
+              if loop_function is not None:
+                prev = output
+          return outputs, state
+
+        def _basic_rnn_seq2seq(encoder_inputs,
+                              decoder_inputs,
+                              cell,
+                              feed_previous,
+                              dtype=dtypes.float32,
+                              scope=None):
+          """Basic RNN sequence-to-sequence model.
+          This model first runs an RNN to encode encoder_inputs into a state vector,
+          then runs decoder, initialized with the last encoder state, on decoder_inputs.
+          Encoder and decoder use the same RNN cell type, but don't share parameters.
+          Args:
+            encoder_inputs: A list of 2D Tensors [batch_size x input_size].
+            decoder_inputs: A list of 2D Tensors [batch_size x input_size].
+            feed_previous: Boolean; if True, only the first of decoder_inputs will be
+              used (the "GO" symbol), all other inputs will be generated by the previous 
+              decoder output using _loop_function below. If False, decoder_inputs are used 
+              as given (the standard decoder case).
+            dtype: The dtype of the initial state of the RNN cell (default: tf.float32).
+            scope: VariableScope for the created subgraph; default: "basic_rnn_seq2seq".
+          Returns:
+            A tuple of the form (outputs, state), where:
+              outputs: A list of the same length as decoder_inputs of 2D Tensors with
+                shape [batch_size x output_size] containing the generated outputs.
+              state: The state of each decoder cell in the final time-step.
+                It is a 2D Tensor of shape [batch_size x cell.state_size].
+          """
+          with variable_scope.variable_scope(scope or "basic_rnn_seq2seq"):
+            enc_cell = copy.deepcopy(cell)
+            _, enc_state = rnn.static_rnn(enc_cell, encoder_inputs, dtype=dtype)
+            if feed_previous:
+                return _rnn_decoder(decoder_inputs, enc_state, cell, _loop_function)
+            else:
+                return _rnn_decoder(decoder_inputs, enc_state, cell)
+
+        def _loop_function(prev, _):
+          '''Naive implementation of loop function for _rnn_decoder. Transform prev from 
+          dimension [batch_size x hidden_dim] to [batch_size x output_dim], which will be
+          used as decoder input of next time step '''
+          return tf.matmul(prev, weights['out']) + biases['out']
+
+        dec_outputs, dec_memory = _basic_rnn_seq2seq(
+            enc_inp, 
+            dec_inp, 
+            cell, 
+            feed_previous = feed_previous
+        )
+
+        reshaped_outputs = [tf.matmul(i, weights['out']) + biases['out'] for i in dec_outputs]
+
+    # Training loss and optimizer
+    with tf.variable_scope('Loss'):
+        # L2 loss
+        output_loss = 0
+        for _y, _Y in zip(reshaped_outputs, target_seq):
+            output_loss += tf.reduce_mean(tf.pow(_y - _Y, 2))
+
+        # L2 regularization for weights and biases
+        reg_loss = 0
+        for tf_var in tf.trainable_variables():
+            if 'Biases_' in tf_var.name or 'Weights_' in tf_var.name:
+                reg_loss += tf.reduce_mean(tf.nn.l2_loss(tf_var))
+
+        loss = output_loss + lambda_l2_reg * reg_loss
+
+    with tf.variable_scope('Optimizer'):
+        optimizer = tf.contrib.layers.optimize_loss(
+                loss=loss,
+                learning_rate=learning_rate,
+                global_step=global_step,
+                optimizer='Adam',
+                clip_gradients=GRADIENT_CLIPPING)
+
+    saver = tf.train.Saver
+
+    return dict(
+        enc_inp = enc_inp, 
+        target_seq = target_seq, 
+        train_op = optimizer, 
+        loss=loss,
+        saver = saver, 
+        reshaped_outputs = reshaped_outputs,
+        )

build_model_with_outliers.py

@@ -1,6 +1,8 @@
+import tensorflow as tf 
 from tensorflow.contrib import rnn
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.framework import dtypes
+import copy
 
 ## Parameters
 learning_rate = 0.01