Keep parameter space noise consistent with action space noise (Fix 51…

…73) (#5193)

*  make parameter space noise consistent with action space noise

*  modified according to lint check

*  indent

python/ray/rllib/agents/ddpg/ddpg_policy.py

@@ -58,7 +58,10 @@ def postprocess_trajectory(self,
             distance_in_action_space = np.sqrt(
                 np.mean(np.square(clean_actions - noisy_actions)))
             self.pi_distance = distance_in_action_space
-            if distance_in_action_space < self.config["exploration_ou_sigma"]:
+            if distance_in_action_space < \
+                    self.config["exploration_ou_sigma"] * self.cur_noise_scale:
+                # multiplying the sampled OU noise by noise scale is
+                # equivalent to multiplying the sigma of OU by noise scale
                 self.parameter_noise_sigma_val *= 1.01
             else:
                 self.parameter_noise_sigma_val /= 1.01