Fixed HER

stable_baselines/her/actor_critic.py

@@ -41,7 +41,7 @@ def __init__(self, inputs_tf, dim_obs, dim_goal, dim_action,
         # Networks.
         with tf.variable_scope('pi'):
             self.pi_tf = self.max_u * tf.tanh(mlp(
-                input_pi, [self.hidden] * self.layers + [self.dimu]))
+                input_pi, [self.hidden] * self.layers + [self.dim_action]))
         with tf.variable_scope('Q'):
             # for policy training
             input_q = tf.concat(axis=1, values=[obs, goals, self.pi_tf / self.max_u])

stable_baselines/her/ddpg.py

@@ -19,7 +19,7 @@ class DDPG(object):
     def __init__(self, input_dims, buffer_size, hidden, layers, network_class, polyak, batch_size,
                  q_lr, pi_lr, norm_eps, norm_clip, max_u, action_l2, clip_obs, scope, time_horizon,
                  rollout_batch_size, subtract_goals, relative_goals, clip_pos_returns, clip_return,
-                 sample_transitions, gamma, reuse=False):
+                 sample_transitions, gamma,  info, use_mpi, reuse=False):
         """
         Implementation of DDPG that is used in combination with Hindsight Experience Replay (HER).
 

stable_baselines/her/experiment/config.py

@@ -125,7 +125,8 @@ def configure_her(params):
     env = cached_make_env(params['make_env'])
     env.reset()
 
-    def reward_fun(achieved_goal, goal, info):  # vectorized
+    def reward_fun(ag_2, g, info):  # vectorized
+        achieved_goal, goal = ag_2, g
         return env.compute_reward(achieved_goal=achieved_goal, desired_goal=goal, info=info)
 
     # Prepare configuration for HER.

stable_baselines/her/experiment/train.py

@@ -22,15 +22,15 @@ def mpi_average(value):
     :param value: (np.ndarray) the array
     :return: (float) the average
     """
-    if len(value) == 0:
-        value = [0.]
     if not isinstance(value, list):
         value = [value]
+    if len(value) == 0:
+        value = [0.]
     return mpi_moments(np.array(value))[0]
 
 
 def train(policy, rollout_worker, evaluator, n_epochs, n_test_rollouts, n_cycles, n_batches, policy_save_interval,
-          save_policies):
+          save_policies=True):
     """
     train the given policy
 
@@ -60,7 +60,7 @@ def train(policy, rollout_worker, evaluator, n_epochs, n_test_rollouts, n_cycles
             episode = rollout_worker.generate_rollouts()
             policy.store_episode(episode)
             for _ in range(n_batches):
-                policy.train_step()
+                policy.train()
             policy.update_target_net()
 
         # test

stable_baselines/her/rollout.py

@@ -8,7 +8,7 @@
 
 
 class RolloutWorker:
-    def __init__(self, make_env, policy, dims, logger, time_horizon, rollout_batch_size=1,
+    def __init__(self, make_env, policy, dims, logger, time_horizon, gamma, rollout_batch_size=1,
                  exploit=False, use_target_net=False, compute_q=False, noise_eps=0,
                  random_eps=0, history_len=100, render=False):
         """
@@ -93,7 +93,7 @@ def generate_rollouts(self):
         achieved_goals[:] = self.initial_ag
 
         # generate episodes
-        obs, achieved_goals, acts, goals, successes = [], [], [], [], []
+        obs, ags, acts, goals, successes = [], [], [], [], []
         info_values = [np.empty((self.time_horizon, self.rollout_batch_size, self.dims['info_' + key]), np.float32)
                        for key in self.info_keys]
         q_values = []
@@ -141,20 +141,20 @@ def generate_rollouts(self):
                 return self.generate_rollouts()
 
             obs.append(observations.copy())
-            achieved_goals.append(achieved_goals.copy())
+            ags.append(achieved_goals.copy())
             successes.append(success.copy())
             acts.append(action.copy())
             goals.append(self.goals.copy())
             observations[...] = o_new
             achieved_goals[...] = ag_new
         obs.append(observations.copy())
-        achieved_goals.append(achieved_goals.copy())
+        ags.append(achieved_goals.copy())
         self.initial_obs[:] = observations
 
         episode = dict(o=obs,
                        u=acts,
                        g=goals,
-                       ag=achieved_goals)
+                       ag=ags)
         for key, value in zip(self.info_keys, info_values):
             episode['info_{}'.format(key)] = value