Skip to content
Branch: master
Find file Copy path
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
131 lines (99 sloc) 4.28 KB
import chainer
from chainer.backends import cuda
from chainer.backends import intel64
from chainer import optimizer
from chainer import types
import typing_extensions as tpe
class CorrectedMomentumSGDHyperparameter(tpe.Protocol):
"""Protocol class for hyperparameter of corrected momentum SGD.
This is only for PEP 544 compliant static type checkers.
lr = None # type: float
momentum = None # type: float
_default_hyperparam = optimizer.Hyperparameter() # type: CorrectedMomentumSGDHyperparameter # NOQA = 0.01
_default_hyperparam.momentum = 0.9
class CorrectedMomentumSGDRule(optimizer.UpdateRule):
"""Update rule for the corrected momentum SGD.
See :class:`~chainer.optimizers.CorrectedMomentumSGD` for the default
values of the hyperparameters.
parent_hyperparam (~chainer.optimizer.Hyperparameter): Hyperparameter
that provides the default values.
lr (float): Learning rate.
momentum (float): Exponential decay rate of the first order moment.
def __init__(self, parent_hyperparam=None, lr=None, momentum=None):
super(CorrectedMomentumSGDRule, self).__init__(
parent_hyperparam or _default_hyperparam)
if lr is not None: = lr
if momentum is not None:
self.hyperparam.momentum = momentum
def init_state(self, param):
with chainer.using_device(param.device):
self.state['v'] = param.device.xp.zeros_like(
# For iDeep
if isinstance(, intel64.mdarray):
self.state['v'] = intel64.ideep.array(
self.state['v'], itype=intel64.ideep.wgt_array)
def update_core_cpu(self, param):
grad = param.grad
if grad is None:
v = self.state['v']
if isinstance(v, intel64.mdarray):
-1, grad) += * v
v *= self.hyperparam.momentum
v -= grad += * v
def update_core_gpu(self, param):
grad = param.grad
if grad is None:
'T grad, T lr, T momentum',
'T param, T v',
'''v = momentum * v - grad;
param += lr * v;''',
grad,, self.hyperparam.momentum,, self.state['v'])
class CorrectedMomentumSGD(optimizer.GradientMethod):
"""Momentum SGD optimizer.
This implements momentum correction discussed in the third section of
`Accurate, Large Minibatch SGD: Training ImageNet in 1 Hour
:class:`~chainer.optimizers.MomentumSGD` implements the equation (10) of
the paper. This optimizer implements the equation (9).
To get better understanding between the two methods,
we show the equivalence between the equation (9) and modification of
the equation (10) that takes momentum correction into account.
First, we set :math:`v_{t} = \\eta_{t} u_t`.
We substitute this relation to the equation (10).
.. math::
v_{t+1} &= m\\frac{\\eta_{t+1}}{\\eta_{t}}v_t + \\eta_{t+1}g_t \\\\
&= m\\frac{\\eta_{t+1}}{\\eta_{t}}\\eta_{t}u_t +
\\eta_{t+1}g_t \\\\
&= \\eta_{t+1}(m u_t + g_t) \\\\
From this result, we derive :math:`u_{t+1} = m u_t + g_t`, which is how
update tensors are calculated by
:class:`~chainer.optimizers.CorrectedMomentumSGD`. Thus, the equivalence
is shown.
lr (float): Learning rate.
momentum (float): Exponential decay rate of the first order moment.
def __init__(self,,
super(CorrectedMomentumSGD, self).__init__() = lr
self.hyperparam.momentum = momentum
lr = optimizer.HyperparameterProxy('lr')
momentum = optimizer.HyperparameterProxy('momentum')
def create_update_rule(self):
return CorrectedMomentumSGDRule(self.hyperparam)
You can’t perform that action at this time.