fix approximate entropy calculation in PPO and A2C (#130)

DLR-RM · Jul 29, 2020 · 8f9aaae · 8f9aaae
1 parent bd2aae0
commit 8f9aaae
Show file tree

Hide file tree

Showing 3 changed files with 4 additions and 3 deletions.
diff --git a/docs/misc/changelog.rst b/docs/misc/changelog.rst
@@ -32,6 +32,7 @@ Bug Fixes:
 - Fix target for updating q values in SAC: the entropy term was not conditioned by terminals states
 - Use ``cloudpickle.load`` instead of ``pickle.load`` in ``CloudpickleWrapper``. (@shwang)
 - Fixed a bug with orthogonal initialization when `bias=False` in custom policy (@rk37)
+- Fixed approximate entropy calculation in PPO and A2C. (@andyshih12)
 
 Deprecations:
 ^^^^^^^^^^^^^
@@ -357,4 +358,4 @@ And all the contributors:
 @Miffyli @dwiel @miguelrass @qxcv @jaberkow @eavelardev @ruifeng96150 @pedrohbtp @srivatsankrishnan @evilsocket
 @MarvineGothic @jdossgollin @SyllogismRXS @rusu24edward @jbulow @Antymon @seheevic @justinkterry @edbeeching
 @flodorner @KuKuXia @NeoExtended @PartiallyTyped @mmcenta @richardwu @kinalmehta @rolandgvc @tkelestemur @mloo3
-@tirafesi @blurLake @koulakis @joeljosephjin @shwang @rk37
+@tirafesi @blurLake @koulakis @joeljosephjin @shwang @rk37 @andyshih12
diff --git a/stable_baselines3/a2c/a2c.py b/stable_baselines3/a2c/a2c.py
@@ -141,7 +141,7 @@ def train(self) -> None:
             # Entropy loss favor exploration
             if entropy is None:
                 # Approximate entropy when no analytical form
-                entropy_loss = -log_prob.mean()
+                entropy_loss = -th.mean(-log_prob)
             else:
                 entropy_loss = -th.mean(entropy)
 

diff --git a/stable_baselines3/ppo/ppo.py b/stable_baselines3/ppo/ppo.py
@@ -198,7 +198,7 @@ def train(self) -> None:
                 # Entropy loss favor exploration
                 if entropy is None:
                     # Approximate entropy when no analytical form
-                    entropy_loss = -log_prob.mean()
+                    entropy_loss = -th.mean(-log_prob)
                 else:
                     entropy_loss = -th.mean(entropy)