fix(trainer): load checkpoint without ckpt_path which doesn't allow f…

…or updated hyperparameters

fixes #41

everyvoice/base_cli/helpers.py

@@ -7,6 +7,7 @@
 import os
 from enum import Enum
 from pathlib import Path
+from tempfile import TemporaryDirectory
 from typing import List, Optional, Union
 
 from loguru import logger
@@ -167,16 +168,29 @@ def train_base_command(
         num_nodes=nodes,
         detect_anomaly=False,  # used for debugging, but triples training time
     )
-    model_obj = model(config)
+
     data = data_module(config)  # type: ignore
     last_ckpt = (
         config.training.finetune_checkpoint
         if config.training.finetune_checkpoint is not None
         and os.path.exists(config.training.finetune_checkpoint)
         else None
     )
-    tensorboard_logger.log_hyperparams(config.dict())
-    trainer.fit(model_obj, data, ckpt_path=last_ckpt)
+    with TemporaryDirectory() as tmpdir:
+        if last_ckpt is not None:
+            import torch
+
+            model_obj = torch.load(last_ckpt)
+            model_obj["hyper_parameters"]["config"].training = config.training  # type: ignore
+            # This is silly but seems to be the most straightforward way of
+            # not losing the current epoch https://github.com/Lightning-AI/lightning/issues/12819#issuecomment-1644018988
+            last_ckpt = Path(tmpdir) / "new.ckpt"
+            # save modified file
+            torch.save(model_obj, last_ckpt)
+            # delete model from memory
+            del model_obj
+        tensorboard_logger.log_hyperparams(config.dict())
+        trainer.fit(model(config), data, ckpt_path=last_ckpt)
 
 
 def inference_base_command(name: Enum):