From 6277c1dc01ecc57ef5e5e35ce99bd8963fe76216 Mon Sep 17 00:00:00 2001 From: Shantanu Acharya Date: Wed, 15 Jun 2022 14:49:57 -0700 Subject: [PATCH 1/3] give option to user to specify wandb save dir via config Signed-off-by: Shantanu Acharya --- examples/asr/conf/asr_adapters/asr_adaptation.yaml | 1 + nemo/utils/exp_manager.py | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/examples/asr/conf/asr_adapters/asr_adaptation.yaml b/examples/asr/conf/asr_adapters/asr_adaptation.yaml index fad6fa981a1a..34f2b4293438 100644 --- a/examples/asr/conf/asr_adapters/asr_adaptation.yaml +++ b/examples/asr/conf/asr_adapters/asr_adaptation.yaml @@ -190,6 +190,7 @@ exp_manager: name: null project: null entity: null + save_dir: null resume_if_exists: false resume_ignore_no_checkpoint: false diff --git a/nemo/utils/exp_manager.py b/nemo/utils/exp_manager.py index 011c29c8317c..f7ccd356e772 100644 --- a/nemo/utils/exp_manager.py +++ b/nemo/utils/exp_manager.py @@ -665,7 +665,10 @@ def configure_loggers( wandb_kwargs = {} if "name" not in wandb_kwargs and "project" not in wandb_kwargs: raise ValueError("name and project are required for wandb_logger") - wandb_logger = WandbLogger(save_dir=exp_dir, version=version, **wandb_kwargs) + + # Update the wandb save_dir + wandb_kwargs['save_dir'] = wandb_kwargs.get('save_dir') or exp_dir + wandb_logger = WandbLogger(version=version, **wandb_kwargs) logger_list.append(wandb_logger) logging.info("WandBLogger has been set up") From 892b4ab81681fd6437d776dd483f25710cf30a7b Mon Sep 17 00:00:00 2001 From: Shantanu Acharya Date: Wed, 15 Jun 2022 15:50:24 -0700 Subject: [PATCH 2/3] create save_dir directory for wandb logger if not exists Signed-off-by: Shantanu Acharya --- nemo/utils/exp_manager.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nemo/utils/exp_manager.py b/nemo/utils/exp_manager.py index f7ccd356e772..6284ef83f328 100644 --- a/nemo/utils/exp_manager.py +++ b/nemo/utils/exp_manager.py @@ -668,6 +668,7 @@ def configure_loggers( # Update the wandb save_dir wandb_kwargs['save_dir'] = wandb_kwargs.get('save_dir') or exp_dir + os.makedirs(wandb_kwargs['save_dir'], exist_ok=True) wandb_logger = WandbLogger(version=version, **wandb_kwargs) logger_list.append(wandb_logger) From 54a4b4e50a635f1d6ba201452a7d1885f300b02c Mon Sep 17 00:00:00 2001 From: Shantanu Acharya Date: Wed, 15 Jun 2022 22:36:05 -0700 Subject: [PATCH 3/3] update save_dir get method with a default value Signed-off-by: Shantanu Acharya --- nemo/utils/exp_manager.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nemo/utils/exp_manager.py b/nemo/utils/exp_manager.py index 6284ef83f328..cf653bb34c87 100644 --- a/nemo/utils/exp_manager.py +++ b/nemo/utils/exp_manager.py @@ -667,7 +667,8 @@ def configure_loggers( raise ValueError("name and project are required for wandb_logger") # Update the wandb save_dir - wandb_kwargs['save_dir'] = wandb_kwargs.get('save_dir') or exp_dir + if wandb_kwargs.get('save_dir', None) is None: + wandb_kwargs['save_dir'] = exp_dir os.makedirs(wandb_kwargs['save_dir'], exist_ok=True) wandb_logger = WandbLogger(version=version, **wandb_kwargs)