Removing in-place updates for batchnorm running variance. This is a m…

…emory leak.
ragavvenkatesan · Mar 3, 2017 · db578fe · db578fe
1 parent cdc13af
commit db578fe
Show file tree

Hide file tree

Showing 3 changed files with 8 additions and 4 deletions.
diff --git a/yann/layers/batch_norm.py b/yann/layers/batch_norm.py
@@ -85,8 +85,9 @@ def __init__ ( self,
 
         mean = theano.tensor.unbroadcast(mean,0)
         var = theano.tensor.unbroadcast(var,0)
+        var = var + 0.000001
         self.updates[self.running_mean] = mean
-        self.updates[self.running_var] = var + 0.001
+        self.updates[self.running_var] = var
 
         self.inference = batch_normalization_test (
                                                 inputs = input,
@@ -237,8 +238,9 @@ def __init__ ( self,
 
         mean = theano.tensor.unbroadcast(mean,0)
         var = theano.tensor.unbroadcast(var,0)
+        var = var + 0.0000001 
         self.updates[self.running_mean] = mean
-        self.updates[self.running_var] = var + 0.001
+        self.updates[self.running_var] = var
 
         self.inference = batch_normalization_test (
                                                 inputs = input,

diff --git a/yann/layers/conv_pool.py b/yann/layers/conv_pool.py
@@ -167,8 +167,9 @@ def __init__ ( self,
 
             mean = theano.tensor.unbroadcast(mean,0)
             var = theano.tensor.unbroadcast(var,0)
+            var = var + 0.000001
             self.updates[self.running_mean] = mean
-            self.updates[self.running_var] = var + 0.001
+            self.updates[self.running_var] = var
 
             batch_norm_inference = batch_normalization_test (
                                                     inputs = pool_out + \

diff --git a/yann/layers/fully_connected.py b/yann/layers/fully_connected.py
@@ -112,8 +112,9 @@ def __init__ (self,
 
             mean = theano.tensor.unbroadcast(mean,0)
             var = theano.tensor.unbroadcast(var,0)
+            var = var + 0. 000001 
             self.updates[self.running_mean] = mean
-            self.updates[self.running_var] = var + 0.001
+            self.updates[self.running_var] = var 
 
             batch_norm_inference = batch_normalization_test (inputs = linear_fit,
                                                             gamma = self.gamma,