[optim] bugfix when all parameters have no grad

test/test_optim.py

@@ -633,6 +633,26 @@ def test_duplicate_params_in_param_group(self):
             self.assertEqual(len(w), 1)
             self.assertIn('a parameter group with duplicate parameters', str(w[0].message))
 
+    def test_no_grad_for_all_params(self):
+        param = torch.randn(5, 5, requires_grad=False)
+
+        optimizer_list = [
+            optim.Adadelta,
+            optim.AdamW,
+            optim.Adam,
+            optim.Adagrad,
+            optim.Adamax,
+            optim.RMSprop,
+            optim.SGD,
+            optim.SparseAdam,
+            optim.ASGD,
+        ]
+        for optim_ctr in optimizer_list:
+            opt = optim_ctr([param, param], lr=0.1)
+            # make sure step can still run even if
+            # all params have no grad
+            opt.step()
+
 
 class SchedulerTestNet(torch.nn.Module):
     def __init__(self):

torch/optim/adadelta.py

@@ -54,6 +54,7 @@ def step(self, closure=None):
             grads = []
             square_avgs = []
             acc_deltas = []
+            lr, rho, eps, weight_decay = group['lr'], group['rho'], group['eps'], group['weight_decay']
 
             for p in group['params']:
                 if p.grad is None:
@@ -74,8 +75,6 @@ def step(self, closure=None):
                 square_avgs.append(state['square_avg'])
                 acc_deltas.append(state['acc_delta'])
 
-                lr, rho, eps, weight_decay = group['lr'], group['rho'], group['eps'], group['weight_decay']
-
                 state['step'] += 1
 
             F.adadelta(params_with_grad,

torch/optim/adam.py

@@ -73,6 +73,7 @@ def step(self, closure=None):
             state_sums = []
             max_exp_avg_sqs = []
             state_steps = []
+            beta1, beta2 = group['betas']
 
             for p in group['params']:
                 if p.grad is not None:
@@ -104,7 +105,6 @@ def step(self, closure=None):
                     # record the step after step update
                     state_steps.append(state['step'])
 
-            beta1, beta2 = group['betas']
             F.adam(params_with_grad,
                    grads,
                    exp_avgs,

torch/optim/adamw.py

@@ -73,6 +73,7 @@ def step(self, closure=None):
             max_exp_avg_sqs = []
             state_steps = []
             amsgrad = group['amsgrad']
+            beta1, beta2 = group['betas']
 
             for p in group['params']:
                 if p.grad is None:
@@ -101,7 +102,6 @@ def step(self, closure=None):
                 if amsgrad:
                     max_exp_avg_sqs.append(state['max_exp_avg_sq'])
 
-                beta1, beta2 = group['betas']
                 # update the steps for each param group update
                 state['step'] += 1
                 # record the step after step update