[RLlib] move evaluation to trainer.step() such that the result is pro…

…perly logged (#12708)

rllib/agents/trainer.py

@@ -535,14 +535,6 @@ def train(self) -> ResultDict:
         if hasattr(self, "workers") and isinstance(self.workers, WorkerSet):
             self._sync_filters_if_needed(self.workers)
 
-        if self.config["evaluation_interval"] == 1 or (
-                self._iteration > 0 and self.config["evaluation_interval"]
-                and self._iteration % self.config["evaluation_interval"] == 0):
-            evaluation_metrics = self._evaluate()
-            assert isinstance(evaluation_metrics, dict), \
-                "_evaluate() needs to return a dict."
-            result.update(evaluation_metrics)
-
         return result
 
     def _sync_filters_if_needed(self, workers: WorkerSet):

rllib/agents/trainer_template.py

@@ -146,6 +146,18 @@ def _init(self, config: TrainerConfigDict,
         @override(Trainer)
         def step(self):
             res = next(self.train_exec_impl)
+
+            # self._iteration gets incremented after this function returns,
+            # meaning that e. g. the first time this function is called,
+            # self._iteration will be 0. We check `self._iteration+1` in the
+            # if-statement below to reflect that the first training iteration
+            # is already over.
+            if (self.config["evaluation_interval"] and (self._iteration + 1) %
+                    self.config["evaluation_interval"] == 0):
+                evaluation_metrics = self._evaluate()
+                assert isinstance(evaluation_metrics, dict), \
+                    "_evaluate() needs to return a dict."
+                res.update(evaluation_metrics)
             return res
 
         @override(Trainer)