pip8

keras/layers/recurrent.py

@@ -707,6 +707,9 @@ def get_config(self):
 class LSTMBN(LSTM):
     '''Long-Short Term Memory unit with Batch Normalization
 
+    This implementation uses the same BN parameters for all steps.
+    The paper recomended using different parameters for each training step.
+
     # Arguments
         LSTM arguments + batch normalization parameters:
         epsilon: small float > 0. Fuzz parameter.
@@ -731,6 +734,7 @@ def __init__(self, output_dim,
                  **kwargs):
         if isinstance(gamma_init, float):
             c = gamma_init
+
             def gamma_init(shape, name=None, c=c):
                 return K.variable(np.ones(shape)*c, name=name)
         self.gamma_init = initializations.get(gamma_init)
@@ -810,13 +814,14 @@ def bn(self, X, fld, slc='i'):
                          axis=reduction_axes)
             std = K.sqrt(std)
             brodcast_std = K.reshape(std, broadcast_shape)
-            mean_update = self.momentum * self.running_mean[fld][slc] + (
-                                                              1 - self.momentum) * m
-            std_update = self.momentum * self.running_std[fld][slc] + (
-                                                            1 - self.momentum) * std
+            mean_update = (self.momentum * self.running_mean[fld][slc] +
+                           (1 - self.momentum) * m)
+            std_update = (self.momentum * self.running_std[fld][slc] +
+                          (1 - self.momentum) * std)
             self.extend_attr('updates',
-                    [(self.running_mean[fld][slc], mean_update),
-                     (self.running_std[fld][slc], std_update)])
+                             [(self.running_mean[fld][slc], mean_update),
+                              (self.running_std[fld][slc], std_update)
+                              ])
 
             X_normed = (X - brodcast_m) / (brodcast_std + self.epsilon)
         else: