checking out some tensorboard changes

keras/backend/tensorflow_backend.py

@@ -1927,9 +1927,11 @@ def __init__(self, inputs, outputs, updates=[]):
                     updates_ops.append(update)
             self.updates_op = tf.group(*updates_ops)
 
-    def __call__(self, inputs):
+    def __call__(self, inputs, **kwargs):
         if not isinstance(inputs, (list, tuple)):
             raise TypeError('`inputs` should be a list or tuple.')
+        unrecognized_kwargs = set(kwargs.keys()) - {'options', 'run_metadata'}
+        assert len(unrecognized_kwargs) == 0, 'Unrecognised kwargs: {}'.format(unrecognized_kwargs)
         feed_dict = {}
         for tensor, value in zip(self.inputs, inputs):
             if is_sparse(tensor):
@@ -1940,7 +1942,7 @@ def __call__(self, inputs):
             feed_dict[tensor] = value
         session = get_session()
         updated = session.run(self.outputs + [self.updates_op],
-                              feed_dict=feed_dict)
+                              feed_dict=feed_dict, **kwargs)
         return updated[:len(self.outputs)]
 
 

keras/backend/theano_backend.py

@@ -14,7 +14,9 @@
     from theano.sandbox.softsign import softsign as T_softsign
 import inspect
 import numpy as np
+import warnings
 from .common import _FLOATX, floatx, _EPSILON, image_dim_ordering
+from .common import _FLOATX, _EPSILON, image_dim_ordering
 py_all = all
 
 
@@ -954,8 +956,14 @@ def __init__(self, inputs, outputs, updates=[], **kwargs):
                                         on_unused_input='ignore',
                                         **kwargs)
 
-    def __call__(self, inputs):
+    def __call__(self, inputs, **kwargs):
         assert isinstance(inputs, (list, tuple))
+        if len(kwargs) > 0:
+            msg = [
+                'Expected no kwargs, you passed %s' % len(kwargs),
+                'kwargs passed to F() are ignored with Theano backend'
+            ]
+            warnings.warn('\n'.join(msg))
         return self.function(*inputs)
 
 

keras/callbacks.py

@@ -118,6 +118,26 @@ def on_batch_end(self, batch, logs=None):
                           'to the batch update (%f). Check your callbacks.'
                           % delta_t_median)
 
+    def on_val_begin(self, epoch, logs=None):
+        """Called at the beginning of validation.
+
+        # Arguments
+            epoch: integer, index of epoch.
+            logs: dictionary of logs.
+        """
+        for callback in self.callbacks:
+            callback.on_val_begin(epoch, logs)
+
+    def on_val_end(self, epoch, logs=None):
+        """Called at the end of validation.
+
+        # Arguments
+            epoch: integer, index of epoch.
+            logs: dictionary of logs.
+        """
+        for callback in self.callbacks:
+            callback.on_val_end(epoch, logs)
+
     def on_train_begin(self, logs=None):
         """Called at the beginning of training.
 
@@ -187,6 +207,12 @@ def on_batch_begin(self, batch, logs=None):
     def on_batch_end(self, batch, logs=None):
         pass
 
+    def on_val_begin(self, epoch, logs=None):
+        pass
+
+    def on_val_end(self, epoch, logs=None):
+        pass
+
     def on_train_begin(self, logs=None):
         pass
 
@@ -237,6 +263,9 @@ def on_epoch_begin(self, epoch, logs=None):
             self.progbar = Progbar(target=self.params['nb_sample'],
                                    verbose=self.verbose)
         self.seen = 0
+        self.epoch_t = None
+        self.batch_t = None
+        self.val_t = None
 
     def on_batch_begin(self, batch, logs=None):
         if self.seen < self.params['nb_sample']:
@@ -256,11 +285,27 @@ def on_batch_end(self, batch, logs=None):
         if self.verbose and self.seen < self.params['nb_sample']:
             self.progbar.update(self.seen, self.log_values)
 
+    def on_val_begin(self, epoch, logs={}):
+        self.epoch_t = self.progbar.elapsed_time()
+        self.batch_t = self.epoch_t / self.params['nb_sample']
+        self.val_start_t = time.time()
+
+    def on_val_end(self, epoch, logs={}):
+        self.val_t = time.time() - self.val_start_t
+
     def on_epoch_end(self, epoch, logs=None):
         logs = logs or {}
         for k in self.params['metrics']:
             if k in logs:
                 self.log_values.append((k, logs[k]))
+        if not self.epoch_t:
+            self.epoch_t = self.progbar.elapsed_time()
+            self.batch_t = self.epoch_t / self.params['nb_sample']
+        # add timings
+        self.log_values.append(('epoch_t', self.epoch_t))
+        self.log_values.append(('batch_t', self.batch_t))
+        if self.val_t:
+            self.log_values.append(('val_t', self.val_t))
         if self.verbose:
             self.progbar.update(self.seen, self.log_values, force=True)
 
@@ -581,7 +626,16 @@ def set_model(self, model):
         self.model = model
         self.sess = K.get_session()
         if self.histogram_freq and self.merged is None:
-            for layer in self.model.layers:
+            def get_layers_flattened(model_layers):
+                layers = []
+                for layer in model_layers:
+                    if layer.__class__.__name__ == 'Model':
+                        layers.extend(get_layers_flattened(layer.layers))
+                    else:
+                        layers.append(layer)
+                return layers
+            layers = get_layers_flattened(self.model.layers)
+            for layer in layers:
 
                 for weight in layer.weights:
                     if hasattr(tf, 'histogram_summary'):
@@ -606,13 +660,17 @@ def set_model(self, model):
                         else:
                             tf.summary.image(weight.name, w_img)
 
-                if hasattr(layer, 'output'):
-                    if hasattr(tf, 'histogram_summary'):
-                        tf.histogram_summary('{}_out'.format(layer.name),
-                                             layer.output)
-                    else:
-                        tf.summary.histogram('{}_out'.format(layer.name),
-                                             layer.output)
+                if layer in self.model.layers:
+                    try:
+                        if hasattr(layer, 'output'):
+                            if hasattr(tf, 'histogram_summary'):
+                                tf.histogram_summary('{}_out'.format(layer.name),
+                                                     layer.output)
+                            else:
+                                tf.summary.histogram('{}_out'.format(layer.name),
+                                                     layer.output)
+                    except AttributeError:
+                        pass
 
         if hasattr(tf, 'merge_all_summaries'):
             self.merged = tf.merge_all_summaries()
@@ -638,21 +696,65 @@ def set_model(self, model):
     def on_epoch_end(self, epoch, logs=None):
         logs = logs or {}
 
-        if self.model.validation_data and self.histogram_freq:
-            if epoch % self.histogram_freq == 0:
-                # TODO: implement batched calls to sess.run
-                # (current call will likely go OOM on GPU)
-                if self.model.uses_learning_phase:
-                    cut_v_data = len(self.model.inputs)
-                    val_data = self.model.validation_data[:cut_v_data] + [0]
-                    tensors = self.model.inputs + [K.learning_phase()]
-                else:
-                    val_data = self.model.validation_data
-                    tensors = self.model.inputs
-                feed_dict = dict(zip(tensors, val_data))
-                result = self.sess.run([self.merged], feed_dict=feed_dict)
-                summary_str = result[0]
+        def get_val_summary(validation_data):
+            if self.model.uses_learning_phase:
+                cut_v_data = len(self.model.inputs)
+                val_data = list(validation_data[:cut_v_data]) + [0]
+                tensors = self.model.inputs + [K.learning_phase()]
+            else:
+                val_data = validation_data
+                tensors = self.model.inputs
+            feed_dict = dict(zip(tensors, val_data))
+            result = self.sess.run([self.merged], feed_dict=feed_dict)
+            return result[0]
+
+        if self.histogram_freq and epoch % self.histogram_freq == 0:
+            if self.model.validation_data:
+                summary_str = get_val_summary(self.model.validation_data)
                 self.writer.add_summary(summary_str, epoch)
+            elif self.model.validation_gen:
+                val_gen = self.model.validation_gen.generator
+                nb_val_samples = self.model.validation_gen.nb_samples
+                # process nb_samples from validation data generator
+                sub_summaries = []
+                processed_samples = 0
+                while processed_samples < nb_val_samples:
+                    validation_data = next(val_gen)
+                    summary = tf.Summary.FromString(get_val_summary(validation_data))
+                    sub_summaries.append(summary)
+                    processed_samples += validation_data[0].shape[0]
+                # convert summaries to dict of lists
+                sub_summaries_dict = {}
+                for sub_summary in sub_summaries:
+                    for value in sub_summary.value:
+                        value_field = value.WhichOneof('value')
+                        value_ifo = sub_summaries_dict.setdefault(value.tag, {'value_field': None, 'values': []})
+                        if not value_ifo['value_field']:
+                            value_ifo['value_field'] = value_field
+                        else:
+                            assert value_ifo['value_field'] == value_field
+                        value_ifo['values'].append(getattr(value, value_field))
+                # aggregate summaries
+                summary = tf.Summary()
+                for name, value_ifo in sub_summaries_dict.items():
+                    summary_value = summary.value.add()
+                    summary_value.tag = name
+                    if value_ifo['value_field'] == 'histo':
+                        values = value_ifo['values']
+                        summary_value.histo.min = min([x.min for x in values])
+                        summary_value.histo.max = max([x.max for x in values])
+                        summary_value.histo.num = sum([x.num for x in values])
+                        summary_value.histo.sum = sum([x.sum for x in values])
+                        summary_value.histo.sum_squares = sum([x.sum_squares for x in values])
+                        # for histogram values, just take first batch for now
+                        # TODO: aggregate histograms over batches
+                        for lim in values[0].bucket_limit:
+                            summary_value.histo.bucket_limit.append(lim)
+                        for bucket in values[0].bucket:
+                            summary_value.histo.bucket.append(bucket)
+                    else:
+                        print('Warning: could not aggregate summary of type {}'.format(value_ifo['value_field']))
+                self.writer.add_summary(summary, epoch)
 
         for name, value in logs.items():
             if name in ['batch', 'size']:

keras/engine/topology.py

@@ -252,6 +252,7 @@ class Layer(object):
         weights: The concatenation of the lists trainable_weights and
             non_trainable_weights (in this order).
         constraints: Dict mapping weights to constraints.
+        multipliers: dict mapping weights to learning rates multipliers.
 
     # Methods
         call(x, mask=None): Where the layer's logic lives.
@@ -310,6 +311,8 @@ def __init__(self, **kwargs):
             self.losses = []
         if not hasattr(self, 'constraints'):
             self.constraints = {}  # dict {tensor: constraint instance}
+        if not hasattr(self, 'multipliers'):
+            self.multipliers = {}  # dict {tensor: multiplier value}
         self.built = False
 
         # These properties should be set by the user via keyword arguments.
@@ -403,7 +406,8 @@ def create_input_layer(self, batch_input_shape,
     def add_weight(self, shape, initializer, name=None,
                    trainable=True,
                    regularizer=None,
-                   constraint=None):
+                   constraint=None,
+                   multiplier=None):
         """Adds a weight variable to the layer.
 
         # Arguments
@@ -420,6 +424,8 @@ def add_weight(self, shape, initializer, name=None,
             self.add_loss(regularizer(weight))
         if constraint is not None:
             self.constraints[weight] = constraint
+        if multiplier is not None:
+            self.multipliers[weight] = multiplier
         if trainable:
             self._trainable_weights.append(weight)
         else:
@@ -1068,6 +1074,7 @@ def __init__(self, input_shape=None, batch_input_shape=None,
         self.inbound_nodes = []
         self.outbound_nodes = []
         self.constraints = {}
+        self.multipliers = {}
         self.sparse = sparse
 
         if not name:
@@ -1275,6 +1282,7 @@ def __init__(self, layers=None, mode='sum', concat_axis=-1,
         self.inbound_nodes = []
         self.outbound_nodes = []
         self.constraints = {}
+        self.multipliers = {}
         self._trainable_weights = []
         self._non_trainable_weights = []
         self.supports_masking = True
@@ -1715,6 +1723,7 @@ class Container(Layer):
         trainable_weights (list of variables)
         non_trainable_weights (list of variables)
         constraints (list of tuples (weight, constraint))
+        multipliers (list of tuples (weight, learning_rate_multiplier))
 
     # Methods
         summary
@@ -2031,6 +2040,7 @@ def build_map_of_graph(tensor, seen_nodes=set(), depth=0,
         self.supports_masking = False
         # The following are implemented as property functions:
         # self.constraints
+        # self.multipliers
         # self.trainable_weights
         # self.non_trainable_weights
         # self.input_spec
@@ -2141,6 +2151,17 @@ def constraints(self):
                 cons[key] = value
         return cons
 
+    @property
+    def multipliers(self):
+        mults = {}
+        for layer in self.layers:
+            for key, value in layer.multipliers.items():
+                if key in mults:
+                    raise Exception('Received multiple learning rate multipliers '
+                                    'for one weight tensor: ' + str(key))
+                mults[key] = value
+        return mults
+
     @property
     def regularizers(self):
         warnings.warn('The `regularizers` attribute of layers/models '