better log for grad & model preparation

README.md

@@ -1,11 +1,11 @@
 # tensorpack
 Neural Network Toolbox on TensorFlow
 
-In development. No document.
+In development. No document. See [examples](https://github.com/ppwwyyxx/tensorpack/tree/master/examples).
 
 ## Features:
 + Scoped abstraction of common models.
-+ Callbacks systems to control different aspects of training.
-+ Use `Dataflow` to gain fine-grained control on data preprocessing.
-+ Training and testing graph are modeled together. Just need to follow the conventions to setup stuffs.
++ Use `Dataflow` to define data preprocessing in pure Python.
++ Callbacks systems to control training.
++ Training and testing are described together. Just need to follow the conventions to setup stuffs.
 + Write summary easier for tensorboard.

tensorpack/callbacks/common.py

@@ -36,7 +36,7 @@ def _get_vars(self):
         for v in vars:
             name = v.op.name
             if re.match('tower[1-9]', name):
-                logger.info("Skip {} when saving model.".format(name))
+                #logger.info("Skip {} when saving model.".format(name))
                 continue
             if 'tower0/' in name:
                 new_name = name.replace('tower0/', '')

tensorpack/callbacks/param.py

@@ -50,7 +50,7 @@ def get_current_value(self):
         ret = self._get_current_value()
         if ret is not None and ret != self.last_value:
             logger.info("{} at epoch {} will change to {}".format(
-                self.op_name, self.epoch_num, ret))
+                self.op_name, self.epoch_num + 1, ret))
         self.last_value = ret
         return ret
 

tensorpack/dataflow/imgaug/crop.py

@@ -36,8 +36,8 @@ def __init__(self, crop_shape):
 
     def _augment(self, img):
         orig_shape = img.arr.shape
-        h0 = (orig_shape[0] - self.crop_shape[0]) * 0.5
-        w0 = (orig_shape[1] - self.crop_shape[1]) * 0.5
+        h0 = int((orig_shape[0] - self.crop_shape[0]) * 0.5)
+        w0 = int((orig_shape[1] - self.crop_shape[1]) * 0.5)
         img.arr = img.arr[h0:h0+self.crop_shape[0],w0:w0+self.crop_shape[1]]
         if img.coords:
             raise NotImplementedError()

tensorpack/train/trainer.py

@@ -88,8 +88,12 @@ def _average_grads(tower_grads):
         ret = []
         with tf.device('/gpu:0'):
             for grad_and_vars in zip(*tower_grads):
-                grad = tf.add_n([x[0] for x in grad_and_vars]) / float(len(tower_grads))
                 v = grad_and_vars[0][1]
+                try:
+                    grad = tf.add_n([x[0] for x in grad_and_vars]) / float(len(tower_grads))
+                except AssertionError:
+                    logger.error("Error while processing gradients of {}".format(v.name))
+                    raise
                 ret.append((grad, v))
             return ret
 
@@ -129,6 +133,7 @@ def get_model_inputs():
                         tf.get_variable_scope().reuse_variables()
                         for k in coll_keys:
                             kept_summaries[k] = copy.copy(tf.get_collection(k))
+                    logger.info("Graph built for tower {}.".format(i))
             for k in coll_keys:
                 del tf.get_collection(k)[:]
                 tf.get_collection(k).extend(kept_summaries[k])