speed monitoring

deepy/dataset/ondisk_dataset.py

@@ -38,9 +38,9 @@ def __init__(self, train_path, valid_path=None, test_path=None, train_size=None,
             logging.info("Cache on memory")
             self._cached_train_data = list(map(self._post_processing, StreamPickler.load(open(self._train_path))))
             self._train_size = len(self._cached_train_data)
-            if self._shuffle_memory:
-                logging.info("Shuffle on-memory data")
-                global_rand.shuffle(self._cached_train_data)
+            # if self._shuffle_memory:
+            #     logging.info("Shuffle on-memory data")
+            #     global_rand.shuffle(self._cached_train_data)
 
     def curriculum_train_data(self):
         self._curriculum_count += 1
@@ -61,6 +61,9 @@ def generate_test_data(self):
 
     def train_set(self):
         if self._cache_on_memory:
+            if self._shuffle_memory:
+                logging.info("shuffle on-memory data")
+                global_rand.shuffle(self._cached_train_data)
             if self._curriculum:
                 if not isinstance(self._curriculum(self._cached_train_data, 1), types.GeneratorType):
                     raise Exception("Curriculum function must be a generator.")

deepy/layers/__init__.py

@@ -15,7 +15,7 @@
 from prelu import PRelu
 from irnn import IRNN
 from lstm import LSTM
-from plstm import PeepholeLSTM
+from plstm import PeepholeLSTM, PLSTM
 from gru import GRU
 from batch_norm import BatchNormalization
 from onehot_embed import OneHotEmbedding

deepy/layers/neural_var.py

@@ -27,7 +27,11 @@ def getitem_wrapper(t, index):
                 index = tuple(index)
             return t.__getitem__(index)
         ret = getitem_wrapper(self, index)
-        ret.output_dim = self.dim()
+        if (hasattr(ret.tensor, 'tag') and hasattr(ret.tensor.tag, 'test_value')
+            and ret.tensor.tag.test_value is not None and len(ret.tensor.tag.test_value.shape) > 0):
+            ret.output_dim = ret.tensor.tag.test_value.shape[-1]
+        else:
+            ret.output_dim = self.dim()
         return ret
 
     def __call__(self, *args, **kwargs):

deepy/layers/plstm.py

@@ -77,4 +77,6 @@ def prepare(self):
                                      self.b_i, self.b_f, self.b_c, self.b_o)
         else:
             self.register_parameters(self.U, self.C,
-                                     self.b_i, self.b_f, self.b_c, self.b_o)
+                                     self.b_i, self.b_f, self.b_c, self.b_o)
+
+PLSTM = PeepholeLSTM

deepy/layers/recurrent.py

@@ -113,13 +113,20 @@ def get_step_inputs(self, input_var, states=None, mask=None, additional_inputs=N
 
         return step_inputs
 
-    def compute(self, input_var, mask=None, additional_inputs=None, steps=None, backward=False, all_states=None, return_all=False):
+    def compute(self, input_var, mask=None, additional_inputs=None, steps=None, backward=False, init_states=None, return_all_states=False):
         if additional_inputs and not self.additional_input_dims:
             self.additional_input_dims = map(lambda var: var.dim(), additional_inputs)
-        return super(RecurrentLayer, self).compute(input_var,
-                mask=mask, additional_inputs=additional_inputs, steps=steps, backward=backward, all_states=all_states, return_all=return_all)
+        result_var = super(RecurrentLayer, self).compute(input_var,
+                                                   mask=mask, additional_inputs=additional_inputs, steps=steps, backward=backward, init_states=init_states, return_all_states=return_all_states)
+        if return_all_states:
+            state_map = {}
+            for k in result_var.tensor:
+                state_map[k] = NeuralVariable(result_var.tensor[k], result_var.test_tensor[k], self.output_dim)
+            return state_map
+        else:
+            return result_var
 
-    def compute_tensor(self, input_var, mask=None, additional_inputs=None, steps=None, backward=False, all_states=None, return_all=False):
+    def compute_tensor(self, input_var, mask=None, additional_inputs=None, steps=None, backward=False, init_states=None, return_all_states=False):
         # prepare parameters
         backward = backward if backward else self._go_backwards
         steps = steps if steps else self._steps
@@ -128,8 +135,8 @@ def compute_tensor(self, input_var, mask=None, additional_inputs=None, steps=Non
             raise Exception("Mask only works with sequence input")
         # get initial states
         init_state_map = self.get_initial_states(input_var)
-        if all_states:
-            for name, val in all_states.items():
+        if init_states:
+            for name, val in init_states.items():
                 if name in init_state_map:
                     init_state_map[name] = val
         # get input sequence map
@@ -154,15 +161,15 @@ def compute_tensor(self, input_var, mask=None, additional_inputs=None, steps=Non
         # return main states
         main_states = retval_map[self.main_state]
         if self._output_type == "one":
-            if return_all:
+            if return_all_states:
                 return_map = {}
                 for name, val in retval_map.items():
                     return_map[name] = val[-1]
                 return return_map
             else:
                 return main_states[-1]
         elif self._output_type == "sequence":
-            if return_all:
+            if return_all_states:
                 return_map = {}
                 for name, val in retval_map.items():
                     return_map[name] = val.dimshuffle((1,0,2))

deepy/trainers/base.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 import sys
+import time
 import numpy as np
 import theano
 
@@ -279,8 +280,11 @@ def train_step(self, train_set, train_size=None):
         network_callback = bool(self.network.training_callbacks)
         trainer_callback = bool(self._iter_callbacks)
         cost_matrix = []
+        exec_count = 0
+        start_time = time.time()
         self._progress = 0
 
+
         for x in train_set:
             if self._skip_batches == 0:
 
@@ -301,6 +305,7 @@ def train_step(self, train_set, train_size=None):
                         cost_x = self.learn(*[t[(t.shape[0]/2):] for t in x])
                 cost_matrix.append(cost_x)
                 self.last_cost = cost_x[0]
+                exec_count += 1
                 if network_callback:
                     self.network.training_callback()
                 if trainer_callback:
@@ -310,7 +315,8 @@ def train_step(self, train_set, train_size=None):
                 self._skip_batches -= 1
             if train_size:
                 self._progress += 1
-                sys.stdout.write("\x1b[2K\r> %d%% | J=%.2f" % (self._progress * 100 / train_size, self.last_cost))
+                spd = float(exec_count) / (time.time() - start_time)
+                sys.stdout.write("\x1b[2K\r> %d%% | J=%.2f | spd=%.2f batch/s" % (self._progress * 100 / train_size, self.last_cost, spd))
                 sys.stdout.flush()
         self._progress = 0