From 4cff8c99f2de53cf15aed7e095d05677953d5bcb Mon Sep 17 00:00:00 2001 From: Sylwester Klocek Date: Wed, 7 Oct 2020 12:14:23 -0700 Subject: [PATCH 1/2] Fix printing momentum for non-deepspeed optimizer Fix printing momentum for non-deepspeed optimizer --- deepspeed/runtime/engine.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/deepspeed/runtime/engine.py b/deepspeed/runtime/engine.py index cbc4aa7e62d6..33433866034a 100755 --- a/deepspeed/runtime/engine.py +++ b/deepspeed/runtime/engine.py @@ -1002,7 +1002,10 @@ def get_type(self): return self._get_optimizer_param('type') def get_mom(self): - return self._get_optimizer_param('betas') + if self.optimizer_name() in DEEPSPEED_OPTIMIZERS: + return self._get_optimizer_param('betas') + else: + return self._get_optimizer_param('momentum') def _report_progress(self, step): lr = self.get_lr() From 71c85a0c9dc98574048424064bea8845ee606120 Mon Sep 17 00:00:00 2001 From: Sylwester Klocek Date: Wed, 7 Oct 2020 14:58:32 -0700 Subject: [PATCH 2/2] fix momentum access for Adam fix momentum access for Adam --- deepspeed/runtime/engine.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deepspeed/runtime/engine.py b/deepspeed/runtime/engine.py index 33433866034a..a442154a03bc 100755 --- a/deepspeed/runtime/engine.py +++ b/deepspeed/runtime/engine.py @@ -1002,10 +1002,10 @@ def get_type(self): return self._get_optimizer_param('type') def get_mom(self): - if self.optimizer_name() in DEEPSPEED_OPTIMIZERS: - return self._get_optimizer_param('betas') - else: + if self.optimizer_name() in ['SGD', 'RMSprop']: return self._get_optimizer_param('momentum') + else: + return self._get_optimizer_param('betas') def _report_progress(self, step): lr = self.get_lr()