stats object counting time wrong

Summary: On each call of the stats.update the object calculates current average iteration time by getting time elapsed from the time_start and then dividing it by the current number of steps. It saves the result to AverageMeter object which when queried returns the average of things saved, so the time is averaged twice which biases it towards the start value (which is often larger).

Reviewed By: kjchalup

Differential Revision: D39206989

fbshipit-source-id: ccab5233d7aaca1ac4fd626fb329b83c7c0d6af9

pytorch3d/implicitron/tools/stats.py

@@ -118,6 +118,7 @@ def __init__(
         self.plot_file = plot_file
         self.do_plot = do_plot
         self.hard_reset(epoch=epoch)
+        self._t_last_update = None
 
     @staticmethod
     def from_json_str(json_str):
@@ -215,7 +216,6 @@ def update(self, preds, time_start=None, freeze_iter=False, stat_set="train"):
             self.it[stat_set] += 1
 
         epoch = self.epoch
-        it = self.it[stat_set]
 
         for stat in self.log_vars:
 
@@ -224,10 +224,11 @@ def update(self, preds, time_start=None, freeze_iter=False, stat_set="train"):
 
             if stat == "sec/it":  # compute speed
                 if time_start is None:
-                    elapsed = 0.0
+                    time_per_it = 0.0
                 else:
-                    elapsed = time.time() - time_start
-                time_per_it = float(elapsed) / float(it + 1)
+                    now = time.time()
+                    time_per_it = now - (self._t_last_update or time_start)
+                    self._t_last_update = now
                 val = time_per_it
             else:
                 if stat in preds: