[rllib] fix clip by value issue as TF upgraded (#4697)

* fix clip_by_value issue * fix typo
ray-project · May 13, 2019 · c5161a2 · c5161a2
1 parent 1622fc2
commit c5161a2
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 5 deletions.
diff --git a/python/ray/rllib/agents/ddpg/ddpg_policy_graph.py b/python/ray/rllib/agents/ddpg/ddpg_policy_graph.py
@@ -166,8 +166,9 @@ def __init__(self, observation_space, action_space, config):
                         stddev=self.config["target_noise"]),
                     -target_noise_clip, target_noise_clip)
                 policy_tp1_smoothed = tf.clip_by_value(
-                    policy_tp1 + clipped_normal_sample, action_space.low,
-                    action_space.high)
+                    policy_tp1 + clipped_normal_sample,
+                    action_space.low * tf.ones_like(policy_tp1),
+                    action_space.high * tf.ones_like(policy_tp1))
             else:
                 # no smoothing, just use deterministic actions
                 policy_tp1_smoothed = policy_tp1
@@ -473,8 +474,9 @@ def make_noisy_actions():
                         tf.shape(deterministic_actions),
                         stddev=self.config["exploration_gaussian_sigma"])
                     stochastic_actions = tf.clip_by_value(
-                        deterministic_actions + normal_sample, action_low,
-                        action_high)
+                        deterministic_actions + normal_sample,
+                        action_low * tf.ones_like(deterministic_actions),
+                        action_high * tf.ones_like(deterministic_actions))
                 elif noise_type == "ou":
                     # add OU noise for exploration, DDPG-style
                     zero_acts = action_low.size * [.0]
@@ -494,7 +496,9 @@ def make_noisy_actions():
                     noise = noise_scale * base_scale \
                         * exploration_value * action_range
                     stochastic_actions = tf.clip_by_value(
-                        deterministic_actions + noise, action_low, action_high)
+                        deterministic_actions + noise,
+                        action_low * tf.ones_like(deterministic_actions),
+                        action_high * tf.ones_like(deterministic_actions))
                 else:
                     raise ValueError(
                         "Unknown noise type '%s' (try 'ou' or 'gaussian')" %

diff --git a/python/ray/rllib/utils/tf_run_builder.py b/python/ray/rllib/utils/tf_run_builder.py
@@ -47,6 +47,8 @@ def get(self, to_fetch):
                     self.session, self.fetches, self.debug_name,
                     self.feed_dict, os.environ.get("TF_TIMELINE_DIR"))
             except Exception:
+                logger.exception("Error fetching: {}, feed_dict={}".format(
+                    self.fetches, self.feed_dict))
                 raise ValueError("Error fetching: {}, feed_dict={}".format(
                     self.fetches, self.feed_dict))
         if isinstance(to_fetch, int):