feat: make dpg, ddpg, pd_ddpg close to offcial implement (#34)

StepNeverStop · Jan 6, 2021 · 3bf7bc0 · 3bf7bc0
1 parent e3b46d9
commit 3bf7bc0
Show file tree

Hide file tree

Showing 4 changed files with 18 additions and 3 deletions.
diff --git a/rls/algos/config.yaml b/rls/algos/config.yaml
@@ -461,6 +461,7 @@ trpo:
 
 dpg:
     gamma: 0.99
+    use_target_action_noise: False
     gaussian_noise_sigma: 0.2 # specify the variance of gaussian distribution
     gaussian_noise_bound: 0.5 # specify the clipping bound of sampled noise, noise must in range of [-bound, bound]
     actor_lr: 5.0e-4
@@ -479,6 +480,7 @@ ddpg:
     gamma: 0.99
     ployak: 0.995
     noise_type: "ou" # ou or gaussian
+    use_target_action_noise: False
     gaussian_noise_sigma: 0.2 # specify the variance of gaussian distribution
     gaussian_noise_bound: 0.5 # specify the clipping bound of sampled noise, noise must in range of [-bound, bound]
     actor_lr: 5.0e-4
@@ -496,6 +498,7 @@ ddpg:
 pd_ddpg:
     gamma: 0.99
     ployak: 0.995
+    use_target_action_noise: False
     gaussian_noise_sigma: 0.2 # specify the variance of gaussian distribution
     gaussian_noise_bound: 0.5 # specify the clipping bound of sampled noise, noise must in range of [-bound, bound]
     actor_lr: 5.0e-4

diff --git a/rls/algos/single/ddpg.py b/rls/algos/single/ddpg.py
@@ -23,6 +23,7 @@ def __init__(self,
 
                  ployak=0.995,
                  noise_type='ou',
+                 use_target_action_noise: False,
                  gaussian_noise_sigma=0.2,
                  gaussian_noise_bound=0.2,
                  actor_lr=5.0e-4,
@@ -38,6 +39,7 @@ def __init__(self,
         self.ployak = ployak
         self.discrete_tau = discrete_tau
         self.noise_type = noise_type
+        self.use_target_action_noise = use_target_action_noise
         self.gaussian_noise_sigma = gaussian_noise_sigma
         self.gaussian_noise_bound = gaussian_noise_bound
 
@@ -131,7 +133,9 @@ def _train(self, BATCH, isw, cell_state):
                 feat_, _ = self._representation_target_net(BATCH.obs_, cell_state=cell_state)
 
                 if self.is_continuous:
-                    action_target = self.target_noised_action(self.ac_target_net.policy_net(feat_))
+                    action_target = self.ac_target_net.policy_net(feat_)
+                    if self.use_target_action_noise:
+                        action_target = self.target_noised_action(action_target)
                     mu = self.ac_net.policy_net(feat)
                 else:
                     target_logits = self.ac_target_net.policy_net(feat_)

diff --git a/rls/algos/single/dpg.py b/rls/algos/single/dpg.py
@@ -26,6 +26,7 @@ def __init__(self,
 
                  actor_lr=5.0e-4,
                  critic_lr=1.0e-3,
+                 use_target_action_noise: False,
                  gaussian_noise_sigma=0.2,
                  gaussian_noise_bound=0.2,
                  discrete_tau=1.0,
@@ -37,6 +38,7 @@ def __init__(self,
                  **kwargs):
         super().__init__(envspec=envspec, **kwargs)
         self.discrete_tau = discrete_tau
+        self.use_target_action_noise = use_target_action_noise
         self.gaussian_noise_sigma = gaussian_noise_sigma
         self.gaussian_noise_bound = gaussian_noise_bound
 
@@ -117,7 +119,9 @@ def _train(self, BATCH, isw, cell_state):
             with tf.GradientTape(persistent=True) as tape:
                 (feat, feat_), _ = self._representation_net(BATCH.obs, cell_state=cell_state, need_split=True)
                 if self.is_continuous:
-                    action_target = self.target_noised_action(self.net.policy_net(feat_))
+                    action_target = self.ac_target_net.policy_net(feat_)
+                    if self.use_target_action_noise:
+                        action_target = self.target_noised_action(action_target)
                     mu = self.net.policy_net(feat)
                 else:
                     target_logits = self.net.policy_net(feat_)

diff --git a/rls/algos/single/pd_ddpg.py b/rls/algos/single/pd_ddpg.py
@@ -28,6 +28,7 @@ def __init__(self,
                  envspec,
 
                  ployak=0.995,
+                 use_target_action_noise: False,
                  gaussian_noise_sigma=0.2,
                  gaussian_noise_bound=0.2,
                  actor_lr=5.0e-4,
@@ -48,6 +49,7 @@ def __init__(self,
         self.discrete_tau = discrete_tau
         self._lambda = tf.Variable(0.0, dtype=tf.float32)
         self.cost_constraint = cost_constraint  # long tern cost <= d
+        self.use_target_action_noise = use_target_action_noise
         self.gaussian_noise_sigma = gaussian_noise_sigma
         self.gaussian_noise_bound = gaussian_noise_bound
 
@@ -147,7 +149,9 @@ def _train(self, BATCH, isw, cell_state):
                 feat_, _ = self._representation_target_net(BATCH.obs_, cell_state=cell_state)
 
                 if self.is_continuous:
-                    action_target = self.target_noised_action(self.ac_target_net.policy_net(feat_))
+                    action_target = self.ac_target_net.policy_net(feat_)
+                    if self.use_target_action_noise:
+                        action_target = self.target_noised_action(action_target)
                     mu = self.ac_net.policy_net(feat)
                 else:
                     target_logits = self.ac_target_net.policy_net(feat_)