Fixed multi-agent open network __done__ bug (#767)

flow/core/kernel/vehicle/traci.py

@@ -68,6 +68,7 @@ def __init__(self,
         # number of vehicles to exit the network for every time-step
         self._num_arrived = []
         self._arrived_ids = []
+        self._arrived_rl_ids = []
 
         # whether or not to automatically color vehicles
         try:
@@ -126,8 +127,11 @@ def update(self, reset):
                 self.kernel_api.vehicle.getSubscriptionResults(veh_id)
         sim_obs = self.kernel_api.simulation.getSubscriptionResults()
 
+        arrived_rl_ids = []
         # remove exiting vehicles from the vehicles class
         for veh_id in sim_obs[tc.VAR_ARRIVED_VEHICLES_IDS]:
+            if veh_id in self.get_rl_ids():
+                arrived_rl_ids.append(veh_id)
             if veh_id in sim_obs[tc.VAR_TELEPORT_STARTING_VEHICLES_IDS]:
                 # this is meant to resolve the KeyError bug when there are
                 # collisions
@@ -137,6 +141,7 @@ def update(self, reset):
             # haven't been removed already
             if vehicle_obs[veh_id] is None:
                 vehicle_obs.pop(veh_id, None)
+        self._arrived_rl_ids.append(arrived_rl_ids)
 
         # add entering vehicles into the vehicles class
         for veh_id in sim_obs[tc.VAR_DEPARTED_VEHICLES_IDS]:
@@ -165,6 +170,7 @@ def update(self, reset):
             self._num_arrived.clear()
             self._departed_ids.clear()
             self._arrived_ids.clear()
+            self._arrived_rl_ids.clear()
 
             # add vehicles from a network template, if applicable
             if hasattr(self.master_kernel.network.network,
@@ -487,6 +493,13 @@ def get_arrived_ids(self):
         else:
             return 0
 
+    def get_arrived_rl_ids(self):
+        """See parent class."""
+        if len(self._arrived_rl_ids) > 0:
+            return self._arrived_rl_ids[-1]
+        else:
+            return 0
+
     def get_departed_ids(self):
         """See parent class."""
         if len(self._departed_ids) > 0:

flow/envs/multiagent/base.py

@@ -121,6 +121,11 @@ def step(self, rl_actions):
         else:
             reward = self.compute_reward(rl_actions, fail=crash)
 
+        for rl_id in self.k.vehicle.get_arrived_rl_ids():
+            done[rl_id] = True
+            reward[rl_id] = 0
+            states[rl_id] = np.zeros(self.observation_space.shape[0])
+
         return states, reward, done, infos
 
     def reset(self, new_inflow_rate=None):