Skip to content

Commit

Permalink
update checkpointing (NVIDIA#2396)
Browse files Browse the repository at this point in the history
Signed-off-by: Jason <jasoli@nvidia.com>
  • Loading branch information
blisc authored and mousebaiker committed Jul 8, 2021
1 parent 42382f6 commit 8986c31
Showing 1 changed file with 9 additions and 1 deletion.
10 changes: 9 additions & 1 deletion nemo/utils/exp_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,8 @@ class CallbackParams:
save_top_k: Optional[int] = 3
save_weights_only: Optional[bool] = False
mode: Optional[str] = "min"
period: Optional[int] = 1
period: Optional[int] = None
every_n_val_epochs: Optional[int] = 1
prefix: Optional[str] = None # If None, exp_manager will attempt to handle the filepath
postfix: str = ".nemo"
save_best_model: bool = False
Expand Down Expand Up @@ -771,6 +772,13 @@ def configure_checkpointing(trainer: 'pytorch_lightning.Trainer', log_dir: Path,
f"{trainer.check_val_every_n_epoch} epochs to ensure that checkpointing will not error out."
)

if params.period is not None:
logging.warning(
"The use of `period` in the checkpoint callback is deprecrated, please use `every_n_val_epochs` instead. "
"Overwriting `every_n_val_epochs` with `period`."
)
params.every_n_val_epochs = params.period

checkpoint_callback = NeMoModelCheckpoint(**params)
checkpoint_callback.last_model_path = trainer.resume_from_checkpoint or ""
trainer.callbacks.append(checkpoint_callback)
Expand Down

0 comments on commit 8986c31

Please sign in to comment.