Close tensorboard's event files properly at the end of the training (#…

…3085)

* add a tensorboard close call at the end of the training

* add tensorboard.close call to the callback trainer

* make pylint happy

* make sphinx happy (I hope)

allennlp/training/callbacks/log_to_tensorboard.py

@@ -107,6 +107,11 @@ def epoch_end_logging(self, trainer: 'CallbackTrainer'):
                                      log_to_console=True,
                                      epoch=trainer.epoch_number + 1)
 
+    @handle_event(Events.TRAINING_END)
+    def training_end(self, trainer: 'CallbackTrainer'):
+        # pylint: disable=unused-argument
+        self.tensorboard.close()
+
     @classmethod
     def from_params(cls, serialization_dir: str, params: Params) -> 'LogToTensorboard':  # type: ignore
         log_batch_size_period = params.pop_int("log_batch_size_period", None)

allennlp/training/tensorboard_writer.py

@@ -211,3 +211,13 @@ def log_activation_histogram(self, outputs, log_prefix: str) -> None:
         else:
             # skip it
             pass
+
+    def close(self) -> None:
+        """
+        Calls the ``close`` method of the ``SummaryWriter`` s which makes sure that pending
+        scalars are flushed to disk and the tensorboard event files are closed properly.
+        """
+        if self._train_log is not None:
+            self._train_log.close()
+        if self._validation_log is not None:
+            self._validation_log.close()

allennlp/training/trainer.py

@@ -549,6 +549,9 @@ def train(self) -> Dict[str, Any]:
 
             epochs_trained += 1
 
+        # make sure pending events are flushed to disk and files are closed properly
+        self._tensorboard.close()
+
         # Load the best model state before returning
         best_model_state = self._checkpointer.best_model_state()
         if best_model_state: