diff --git a/chainer/optimizer.py b/chainer/optimizer.py
index a19b82b15338..82e77a321adc 100644
--- a/chainer/optimizer.py
+++ b/chainer/optimizer.py
@@ -484,9 +484,11 @@ class GradientMethod(Optimizer):
 
     """
 
-    def __init__(self):
+    def __init__(self, link=None):
         super(GradientMethod, self).__init__()
         self.hyperparam = Hyperparameter()
+        if isinstance(link, link_module.Link):
+            self.setup(link)
 
     def setup(self, link):
         super(GradientMethod, self).setup(link)
diff --git a/chainer/optimizers/ada_delta.py b/chainer/optimizers/ada_delta.py
index ef41bc826c7d..096e9a3933a0 100644
--- a/chainer/optimizers/ada_delta.py
+++ b/chainer/optimizers/ada_delta.py
@@ -84,8 +84,8 @@ class AdaDelta(optimizer.GradientMethod):
     """
 
     def __init__(self, rho=_default_hyperparam.rho,
-                 eps=_default_hyperparam.eps):
-        super(AdaDelta, self).__init__()
+                 eps=_default_hyperparam.eps, model=None):
+        super(AdaDelta, self).__init__(model)
         self.hyperparam.rho = rho
         self.hyperparam.eps = eps
 
diff --git a/chainer/optimizers/ada_grad.py b/chainer/optimizers/ada_grad.py
index fd3120c622fc..0b51c2ce1bd1 100644
--- a/chainer/optimizers/ada_grad.py
+++ b/chainer/optimizers/ada_grad.py
@@ -74,8 +74,9 @@ class AdaGrad(optimizer.GradientMethod):
 
     """
 
-    def __init__(self, lr=_default_hyperparam.lr, eps=_default_hyperparam.eps):
-        super(AdaGrad, self).__init__()
+    def __init__(self, lr=_default_hyperparam.lr,
+                 eps=_default_hyperparam.eps, model=None):
+        super(AdaGrad, self).__init__(model)
         self.hyperparam.lr = lr
         self.hyperparam.eps = eps
 
diff --git a/chainer/optimizers/adam.py b/chainer/optimizers/adam.py
index d963beb7f0e3..d680f9632e97 100644
--- a/chainer/optimizers/adam.py
+++ b/chainer/optimizers/adam.py
@@ -110,8 +110,9 @@ def __init__(self,
                  alpha=_default_hyperparam.alpha,
                  beta1=_default_hyperparam.beta1,
                  beta2=_default_hyperparam.beta2,
-                 eps=_default_hyperparam.eps):
-        super(Adam, self).__init__()
+                 eps=_default_hyperparam.eps,
+                 model=None):
+        super(Adam, self).__init__(model)
         self.hyperparam.alpha = alpha
         self.hyperparam.beta1 = beta1
         self.hyperparam.beta2 = beta2
diff --git a/chainer/optimizers/momentum_sgd.py b/chainer/optimizers/momentum_sgd.py
index 2ac275aefa18..a996c21181fd 100644
--- a/chainer/optimizers/momentum_sgd.py
+++ b/chainer/optimizers/momentum_sgd.py
@@ -69,8 +69,8 @@ class MomentumSGD(optimizer.GradientMethod):
     """
 
     def __init__(self, lr=_default_hyperparam.lr,
-                 momentum=_default_hyperparam.momentum):
-        super(MomentumSGD, self).__init__()
+                 momentum=_default_hyperparam.momentum, model=None):
+        super(MomentumSGD, self).__init__(model)
         self.hyperparam.lr = lr
         self.hyperparam.momentum = momentum
 
diff --git a/chainer/optimizers/nesterov_ag.py b/chainer/optimizers/nesterov_ag.py
index 440874586be7..c1626821e9df 100644
--- a/chainer/optimizers/nesterov_ag.py
+++ b/chainer/optimizers/nesterov_ag.py
@@ -76,8 +76,8 @@ class NesterovAG(optimizer.GradientMethod):
     """
 
     def __init__(self, lr=_default_hyperparam.lr,
-                 momentum=_default_hyperparam.momentum):
-        super(NesterovAG, self).__init__()
+                 momentum=_default_hyperparam.momentum, model=None):
+        super(NesterovAG, self).__init__(model)
         self.hyperparam.lr = lr
         self.hyperparam.momentum = momentum
 
diff --git a/chainer/optimizers/rmsprop.py b/chainer/optimizers/rmsprop.py
index 0579b7e7ddf6..35c763e2f3f0 100644
--- a/chainer/optimizers/rmsprop.py
+++ b/chainer/optimizers/rmsprop.py
@@ -91,8 +91,9 @@ class RMSprop(optimizer.GradientMethod):
     """
 
     def __init__(self, lr=_default_hyperparam.lr,
-                 alpha=_default_hyperparam.alpha, eps=_default_hyperparam.eps):
-        super(RMSprop, self).__init__()
+                 alpha=_default_hyperparam.alpha, eps=_default_hyperparam.eps,
+                 model=None):
+        super(RMSprop, self).__init__(model)
         self.hyperparam.lr = lr
         self.hyperparam.alpha = alpha
         self.hyperparam.eps = eps
diff --git a/chainer/optimizers/rmsprop_graves.py b/chainer/optimizers/rmsprop_graves.py
index 598cead73887..06bd4dbf3fdb 100644
--- a/chainer/optimizers/rmsprop_graves.py
+++ b/chainer/optimizers/rmsprop_graves.py
@@ -102,8 +102,9 @@ class RMSpropGraves(optimizer.GradientMethod):
     def __init__(self, lr=_default_hyperparam.lr,
                  alpha=_default_hyperparam.alpha,
                  momentum=_default_hyperparam.momentum,
-                 eps=_default_hyperparam.eps):
-        super(RMSpropGraves, self).__init__()
+                 eps=_default_hyperparam.eps,
+                 model=None):
+        super(RMSpropGraves, self).__init__(model)
         self.hyperparam.lr = lr
         self.hyperparam.alpha = alpha
         self.hyperparam.momentum = momentum
diff --git a/chainer/optimizers/sgd.py b/chainer/optimizers/sgd.py
index 3af829a8c925..b181a786ec3a 100644
--- a/chainer/optimizers/sgd.py
+++ b/chainer/optimizers/sgd.py
@@ -50,8 +50,8 @@ class SGD(optimizer.GradientMethod):
 
     """
 
-    def __init__(self, lr=_default_hyperparam.lr):
-        super(SGD, self).__init__()
+    def __init__(self, lr=_default_hyperparam.lr, model=None):
+        super(SGD, self).__init__(model)
         self.hyperparam.lr = lr
 
     lr = optimizer.HyperparameterProxy('lr')
diff --git a/chainer/optimizers/smorms3.py b/chainer/optimizers/smorms3.py
index bdc79b946429..ccd57f17e755 100644
--- a/chainer/optimizers/smorms3.py
+++ b/chainer/optimizers/smorms3.py
@@ -87,8 +87,9 @@ class SMORMS3(optimizer.GradientMethod):
 
     """
 
-    def __init__(self, lr=_default_hyperparam.lr, eps=_default_hyperparam.eps):
-        super(SMORMS3, self).__init__()
+    def __init__(self, lr=_default_hyperparam.lr,
+                 eps=_default_hyperparam.eps, model=None):
+        super(SMORMS3, self).__init__(model)
         self.hyperparam.lr = lr
         self.hyperparam.eps = eps