ref: remove on_eval_start hook (Lightning-AI#3176)

* remove on_eval_start hook

* remove on_eval_start hook

Author: williamFalcon

pytorch_lightning/callbacks/base.py

@@ -137,18 +137,10 @@ def on_pretrain_routine_end(self, trainer, pl_module):
         """Called when the pretrain routine ends."""
         pass
 
-    def on_validation_start(self, trainer, pl_module):
-        """Called when the validation loop begins."""
-        pass
-
     def on_validation_end(self, trainer, pl_module):
         """Called when the validation loop ends."""
         pass
 
-    def on_test_start(self, trainer, pl_module):
-        """Called when the test begins."""
-        pass
-
     def on_test_end(self, trainer, pl_module):
         """Called when the test ends."""
         pass

pytorch_lightning/callbacks/progress.py

@@ -155,13 +155,13 @@ def on_epoch_start(self, trainer, pl_module):
     def on_train_batch_end(self, trainer, pl_module, batch, batch_idx, dataloader_idx):
         self._train_batch_idx += 1
 
-    def on_validation_start(self, trainer, pl_module):
+    def on_validation_epoch_start(self, trainer, pl_module):
         self._val_batch_idx = 0
 
     def on_validation_batch_end(self, trainer, pl_module, batch, batch_idx, dataloader_idx):
         self._val_batch_idx += 1
 
-    def on_test_start(self, trainer, pl_module):
+    def on_test_epoch_start(self, trainer, pl_module):
         self._test_batch_idx = 0
 
     def on_test_batch_end(self, trainer, pl_module, batch, batch_idx, dataloader_idx):
@@ -338,8 +338,8 @@ def on_train_batch_end(self, trainer, pl_module, batch, batch_idx, dataloader_id
             self.main_progress_bar.update(self.refresh_rate)
             self.main_progress_bar.set_postfix(trainer.progress_bar_dict)
 
-    def on_validation_start(self, trainer, pl_module):
-        super().on_validation_start(trainer, pl_module)
+    def on_validation_epoch_start(self, trainer, pl_module):
+        super().on_validation_epoch_start(trainer, pl_module)
         self.val_progress_bar = self.init_validation_tqdm()
         self.val_progress_bar.total = convert_inf(self.total_val_batches)
 
@@ -358,8 +358,8 @@ def on_train_end(self, trainer, pl_module):
         super().on_train_end(trainer, pl_module)
         self.main_progress_bar.close()
 
-    def on_test_start(self, trainer, pl_module):
-        super().on_test_start(trainer, pl_module)
+    def on_test_epoch_start(self, trainer, pl_module):
+        super().on_test_epoch_start(trainer, pl_module)
         self.test_progress_bar = self.init_test_tqdm()
         self.test_progress_bar.total = convert_inf(self.total_test_batches)
 

pytorch_lightning/trainer/callback_hook.py

@@ -165,21 +165,11 @@ def on_test_batch_end(self, batch, batch_idx, dataloader_idx):
         for callback in self.callbacks:
             callback.on_test_batch_end(self, self.get_model(), batch, batch_idx, dataloader_idx)
 
-    def on_validation_start(self):
-        """Called when the validation loop begins."""
-        for callback in self.callbacks:
-            callback.on_validation_start(self, self.get_model())
-
     def on_validation_end(self):
         """Called when the validation loop ends."""
         for callback in self.callbacks:
             callback.on_validation_end(self, self.get_model())
 
-    def on_test_start(self):
-        """Called when the test begins."""
-        for callback in self.callbacks:
-            callback.on_test_start(self, self.get_model())
-
     def on_test_end(self):
         """Called when the test ends."""
         for callback in self.callbacks:

pytorch_lightning/trainer/evaluate_loop.py

@@ -45,12 +45,6 @@ def should_skip_evaluation(self, dataloaders, max_batches):
 
         return False
 
-    def on_evaluation_start(self, *args, **kwargs):
-        if self.testing:
-            self.trainer.call_hook('on_test_start', *args, **kwargs)
-        else:
-            self.trainer.call_hook('on_validation_start', *args, **kwargs)
-
     def on_evaluation_end(self, *args, **kwargs):
         if self.testing:
             self.trainer.call_hook('on_test_end', *args, **kwargs)

pytorch_lightning/trainer/evaluation_loop.py

@@ -186,9 +186,7 @@ class TrainerEvaluationLoopMixin(ABC):
     on_validation_batch_end: Callable
     on_test_batch_start: Callable
     on_test_batch_end: Callable
-    on_validation_start: Callable
     on_validation_end: Callable
-    on_test_start: Callable
     on_test_end: Callable
     accelerator_backend: ...
     evaluation_loop: EvaluationLoop
@@ -311,8 +309,10 @@ def run_evaluation(self, test_mode: bool = False):
         # set up the loop for val/test
         self.evaluation_loop.testing = test_mode
 
-        # TODO: deprecate
+        # enable eval mode + no grads
         model = self.get_model()
+
+        # TODO: deprecate
         model.on_pre_performance_check()
 
         # select dataloaders
@@ -322,13 +322,9 @@ def run_evaluation(self, test_mode: bool = False):
         if self.evaluation_loop.should_skip_evaluation(dataloaders, max_batches):
             return [], []
 
-        # TODO: deprecate
-        self.evaluation_loop.on_evaluation_start()
-
         # ------------------------------
         # ------------------------------
         # ------------------------------
-        # enable eval mode + no grads
         model.zero_grad()
         model.eval()
         torch.set_grad_enabled(False)

tests/callbacks/test_callbacks.py

@@ -38,9 +38,9 @@ def __init__(self):
             self.on_train_end_called = False
             self.on_pretrain_routine_start_called = False
             self.on_pretrain_routine_end_called = False
-            self.on_validation_start_called = False
+            self.on_validation_epoch_start_called = False
             self.on_validation_end_called = False
-            self.on_test_start_called = False
+            self.on_test_epoch_start_called = False
             self.on_test_end_called = False
 
         def setup(self, trainer, pl_module, stage: str):
@@ -131,17 +131,17 @@ def on_pretrain_routine_end(self, trainer, pl_module):
             _check_args(trainer, pl_module)
             self.on_pretrain_routine_end_called = True
 
-        def on_validation_start(self, trainer, pl_module):
+        def on_validation_epoch_start(self, trainer, pl_module):
             _check_args(trainer, pl_module)
-            self.on_validation_start_called = True
+            self.on_validation_epoch_start_called = True
 
         def on_validation_end(self, trainer, pl_module):
             _check_args(trainer, pl_module)
             self.on_validation_end_called = True
 
-        def on_test_start(self, trainer, pl_module):
+        def on_test_epoch_start(self, trainer, pl_module):
             _check_args(trainer, pl_module)
-            self.on_test_start_called = True
+            self.on_test_epoch_start_called = True
 
         def on_test_end(self, trainer, pl_module):
             _check_args(trainer, pl_module)
@@ -180,9 +180,9 @@ def on_test_end(self, trainer, pl_module):
     assert not test_callback.on_train_end_called
     assert not test_callback.on_pretrain_routine_start_called
     assert not test_callback.on_pretrain_routine_end_called
-    assert not test_callback.on_validation_start_called
+    assert not test_callback.on_validation_epoch_start_called
     assert not test_callback.on_validation_end_called
-    assert not test_callback.on_test_start_called
+    assert not test_callback.on_test_epoch_start_called
     assert not test_callback.on_test_end_called
 
     # fit model
@@ -211,9 +211,9 @@ def on_test_end(self, trainer, pl_module):
     assert not test_callback.on_train_end_called
     assert not test_callback.on_pretrain_routine_start_called
     assert not test_callback.on_pretrain_routine_end_called
-    assert not test_callback.on_validation_start_called
+    assert not test_callback.on_validation_epoch_start_called
     assert not test_callback.on_validation_end_called
-    assert not test_callback.on_test_start_called
+    assert not test_callback.on_test_epoch_start_called
     assert not test_callback.on_test_end_called
 
     trainer.fit(model)
@@ -238,11 +238,11 @@ def on_test_end(self, trainer, pl_module):
     assert test_callback.on_train_end_called
     assert test_callback.on_pretrain_routine_start_called
     assert test_callback.on_pretrain_routine_end_called
-    assert test_callback.on_validation_start_called
+    assert test_callback.on_validation_epoch_start_called
     assert test_callback.on_validation_end_called
     assert not test_callback.on_test_batch_start_called
     assert not test_callback.on_test_batch_end_called
-    assert not test_callback.on_test_start_called
+    assert not test_callback.on_test_epoch_start_called
     assert not test_callback.on_test_end_called
 
     # reset setup teardown callback
@@ -258,9 +258,9 @@ def on_test_end(self, trainer, pl_module):
     assert test_callback.teardown_called
     assert test_callback.on_test_batch_start_called
     assert test_callback.on_test_batch_end_called
-    assert test_callback.on_test_start_called
+    assert test_callback.on_test_epoch_start_called
     assert test_callback.on_test_end_called
-    assert not test_callback.on_validation_start_called
+    assert not test_callback.on_validation_epoch_start_called
     assert not test_callback.on_validation_end_called
     assert not test_callback.on_validation_batch_end_called
     assert not test_callback.on_validation_batch_start_called

tests/trainer/test_dataloaders.py

@@ -704,12 +704,12 @@ def on_train_start(self, trainer, pl_module):
         assert isinstance(train_sampler, DistributedSampler)
         assert train_sampler.shuffle
 
-    def on_validation_start(self, trainer, pl_module):
+    def on_validation_epoch_start(self, trainer, pl_module):
         val_sampler = trainer.val_dataloaders[0].sampler
         assert isinstance(val_sampler, DistributedSampler)
         assert not val_sampler.shuffle
 
-    def on_test_start(self, trainer, pl_module):
+    def on_test_epoch_start(self, trainer, pl_module):
         test_sampler = trainer.test_dataloaders[0].sampler
         assert isinstance(test_sampler, DistributedSampler)
         assert not test_sampler.shuffle