Many objective layers in cook.

docs/source/conf.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python2.7
 # -*- coding: utf-8 -*-
 #
 # yann documentation build configuration file, created by
@@ -20,21 +19,15 @@
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 sys.path.insert(0, os.path.abspath('../..'))
-sys.path.insert(0, os.path.abspath('../../yann/'))
-sys.path.insert(0, os.path.abspath('../../pantry/'))
-
 
 # fool rtd into thinking a GPU is available, so all modules are importable
 try:
     from unittest.mock import Mock
 except ImportError:
     from mock import Mock
 
-# Don't build or import any modules.
-MOCK_MODULES = ['numpy', 'scipy', 'networkx', 'theano', 'progressbar', 'skdata', 'pillow',
- 'matplotlib']
-for mod_name in MOCK_MODULES:
-    sys.modules[mod_name] = Mock()
+import theano
+# import theano.sandbox.cuda
 
 theano.config = Mock(device='gpu')
 # theano.sandbox.cuda.cuda_enabled = True
@@ -74,8 +67,8 @@
 
 # General substitutions.
 project = u'Yann'
-copyright = u'2015-2017, Ragav Venkatesan'
-author = u'Ragav Venkatesan'
+copyright = u'2015-2017, Ragav Venkatesan and Other Yann Contributors'
+author = u'Ragav Venkatesan and Other Yann Contributors.'
 license = u'MIT License'
 
 import yann
@@ -89,9 +82,6 @@
 # The full version, including alpha/beta/rc tags.
 release = yann.__version__
 
-copyright = u'2015–2017, Ragav Venkatesan'
-
-
 # There are two options for replacing |today|: either, you set today to some
 # non-false value, then it is used:
 #today = ''

pantry/tutorials/autoencoder.py

@@ -118,7 +118,7 @@ def autoencoder ( dataset= None, verbose = 1 ):
                     )          
 
     learning_rates = (0, 0.1, 0.01)  
-    net.cook( objective_layer = 'obj',
+    net.cook( objective_layers = ['obj'],
               datastream = 'data',
               learning_rates = learning_rates,
               verbose = verbose

pantry/tutorials/lenet.py

@@ -70,7 +70,7 @@ def lenet5 ( dataset= None, verbose = 1 ):
                     filter_size = (5,5),
                     pool_size = (2,2),
                     activation = 'relu',
-                    regularize = True,
+                    # regularize = True,
                     verbose = verbose
                     )
 
@@ -81,7 +81,7 @@ def lenet5 ( dataset= None, verbose = 1 ):
                     filter_size = (3,3),
                     pool_size = (2,2),
                     activation = 'relu',
-                    regularize = True,
+                    # regularize = True,
                     verbose = verbose
                     )      
 
@@ -91,7 +91,7 @@ def lenet5 ( dataset= None, verbose = 1 ):
                     id = "dot_product_1",
                     num_neurons = 1250,
                     activation = 'relu',
-                    regularize = True,
+                    # regularize = True,
                     verbose = verbose
                     )
 
@@ -100,15 +100,15 @@ def lenet5 ( dataset= None, verbose = 1 ):
                     id = "dot_product_2",
                     num_neurons = 1250,                    
                     activation = 'relu',  
-                    regularize = True,    
+                    # regularize = True,    
                     verbose = verbose
                     ) 
 
     net.add_layer ( type = "classifier",
                     id = "softmax",
                     origin = "dot_product_2",
                     num_classes = 10,
-                    regularize = True,
+                    # regularize = True,
                     activation = 'softmax',
                     verbose = verbose
                     )
@@ -126,14 +126,9 @@ def lenet5 ( dataset= None, verbose = 1 ):
     net.pretty_print()  
     draw_network(net.graph, filename = 'lenet.png')   
 
-    net.cook( optimizer = 'main',
-              objective_layer = 'obj',
-              datastream = 'data',
-              classifier_layer = 'softmax',
-              verbose = verbose
-              )
+    net.cook()
 
-    net.train( epochs = (40, 40), 
+    net.train( epochs = (4, 4), 
                validate_after_epochs = 1,
                training_accuracy = True,
                learning_rates = learning_rates,               
@@ -268,16 +263,11 @@ def lenet_maxout_batchnorm_before_activation ( dataset= None, verbose = 1 ):
 
     learning_rates = (0.05, 0.001, 0.0001)  
 
-    net.cook( optimizer = 'main',
-              objective_layer = 'obj',
-              datastream = 'data',
-              classifier = 'softmax',
-              verbose = verbose
-              )
+    net.cook(  )
     #draw_network(net.graph, filename = 'lenet.png')    
     net.pretty_print()
 
-    net.train( epochs = (40, 40), 
+    net.train( epochs = (4, 4), 
                validate_after_epochs = 1,
                visualize_after_epochs = 1,
                training_accuracy = True,
@@ -432,16 +422,11 @@ def lenet_maxout_batchnorm_after_activation ( dataset= None, verbose = 1 ):
 
     learning_rates = (0.05, 0.001, 0.0001)  
 
-    net.cook( optimizer = 'main',
-              objective_layer = 'obj',
-              datastream = 'data',
-              classifier = 'softmax',
-              verbose = verbose
-              )
+    net.cook( )
     draw_network(net.graph, filename = 'lenet.png')    
     net.pretty_print()
 
-    net.train( epochs = (40, 40), 
+    net.train( epochs = (4, 4), 
                validate_after_epochs = 1,
                visualize_after_epochs = 1,
                training_accuracy = True,
@@ -475,5 +460,5 @@ def lenet_maxout_batchnorm_after_activation ( dataset= None, verbose = 1 ):
         dataset = data.dataset_location()
 
     lenet5 ( dataset, verbose = 2 )
-    # lenet_maxout_batchnorm_before_activation (dataset, verbose = 2)
-    # lenet_maxout_batchnorm_after_activation (dataset, verbose = 2)
+    lenet_maxout_batchnorm_before_activation (dataset, verbose = 2)
+    lenet_maxout_batchnorm_after_activation (dataset, verbose = 2)

pantry/tutorials/mlp.py

@@ -72,7 +72,6 @@ def mlp ( dataset, verbose = 1 ):
     learning_rates = (0.05, 0.01, 0.001)  
 
     net.cook( optimizer = 'main',
-              objective_layer = 'obj',
               datastream = 'data',
               classifier = 'softmax',
               verbose = verbose

yann/network.py

@@ -1647,7 +1647,7 @@ def _initialize_test (self, verbose = 2, **kwargs):
             datastream: as always
             classifier: the classifier layer to test out of.
             generator: if generator network, generator layer to test out of.
-            value: if value-based objective network, the value layer to test out of.
+            value: if value-based objective network, the value to test out of.
             verbose: as always
 
         """
@@ -1662,7 +1662,7 @@ def _initialize_test (self, verbose = 2, **kwargs):
             self._initialize_test_classifier(errors = _errors, verbose = verbose)
 
         elif 'value' in kwargs.keys():
-            _errors = self.inference_layers[kwargs['value']].inference
+            _errors = kwargs['value']
             self._initialize_test_value(errors = _errors, verbose = verbose)
 
         else:
@@ -2107,8 +2107,10 @@ def cook(self, verbose = 2, **kwargs):
                           Default is the last visualizer created.
             classifier_layer: supply the layer of classifier.
                           Default is the last classifier layer created.
-            objective_layer: Supply the layer id of layer that has the objective function.
+            objective_layers: Supply a list of layer ids of layers that has the objective function.
                           Default is last objective layer created if no classifier is provided.
+            objective_weights: Supply a list of weights to be multiplied by each value of the 
+                         objective layers. Default is 1.
             active_layers: Supply a list of active layers. If this parameter is supplied all
                            ``'learnabile'`` of all layers will be ignored and only these layers
                            will be trained. By default, all the learnable layers are used.
@@ -2156,10 +2158,15 @@ def cook(self, verbose = 2, **kwargs):
                 classifier_layer = None
                 self.network_type = 'value'
 
-        if not 'objective_layer' in kwargs.keys():
-            objective_layer = self.last_objective_layer_created
+        if not 'objective_layers' in kwargs.keys():
+            objective_layers = [self.last_objective_layer_created]
         else:
-            objective_layer = kwargs['objective_layer']
+            objective_layers = kwargs['objective_layers']
+
+        if not 'objective_weights' in kwargs.keys():
+            objective_weights = [1]*len(objective_layers)            
+        else:
+            objective_weights = kwargs['objective_weights']
 
         if not 'active_layers' in kwargs.keys():
             params = self.active_params
@@ -2221,21 +2228,28 @@ def cook(self, verbose = 2, **kwargs):
                 raise Exception ("Datastream " + datastream + " not found.")
         self.cooked_datastream = self.datastream[datastream]
 
-        if objective_layer is None:
+        if objective_layers is None:
             if self.last_objective_layer_created is None:
                 raise Exception ("Cannot build trainer without having an objective layer created")
             else:
-                objective_layer = self.last_objective_layer_created
+                objective_layers = self.last_objective_layer_created
+
+        try:
+            assert len(objective_layers) == len(objective_weights)
+        except:
+            raise Exception (" Number of objective weights and objective layers must match.")
 
         if verbose >=2 :
             print (".. All checks complete, cooking continues" )
 
-        self.cost = self.layers[objective_layer].output
-        self.dropout_cost = self.dropout_layers[objective_layer].output
-        self.cost = []
+        self.layer_cost = 0
+        self.dropout_cost = 0
+        for lyr, weight in zip(objective_layers, objective_weights):
+            self.layer_cost = self.layer_cost + weight * self.layers[lyr].output
+            self.dropout_cost = self.dropout_cost + weight * self.dropout_layers[lyr].output
 
+        self.cost = []
         self._cook_datastream(verbose = verbose)
-
         self._cook_optimizer(params = params,
                              optimizer = self.cooked_optimizer,
                              objective = self.dropout_cost,
@@ -2249,9 +2263,10 @@ def cook(self, verbose = 2, **kwargs):
             self._initialize_posterior (classifier = classifier_layer,
                                    verbose = verbose)
         else:
-            self._initialize_test (value = objective_layer,
+            self._initialize_test (value = self.layer_cost,
                                    verbose = verbose)
-        self._initialize_train ( verbose = verbose )
+        self._initialize_train ( objective = self.dropout_cost,
+                                 verbose = verbose )
 
         self._cook_resultor(resultor = self.cooked_resultor, verbose = verbose)
         self._cook_visualizer(verbose = verbose) # always cook visualizer last.