Merge pull request #458 from BDonnot/master

Add counts to high resolution simulator

CHANGELOG.rst

@@ -96,7 +96,11 @@ Change Log
   `obs.get_env_from_external_forecasts(...)`
 - [ADDED] adding the `TimedOutEnvironment` that takes "do nothing" actions when the agent
   takes too much time to compute. This involves quite some changes in the runner too.
-- [IMPROVED] possibility to "chain" the call to simulate when multiple forecast
+- [ADDED] experimental support to count the number of "high resolution simulator" (`obs.simulate`, 
+  `obs.get_simulator` and `obs.get_forecast_env`) in the environment (see 
+  https://github.com/rte-france/Grid2Op/issues/417). It might not work properly in distributed settings
+  (if the agents uses parrallel processing or if MultiProcessEnv is used), in MultiMixEnv, etc.
+- [IMPROVED] possibility to "chain" the call to simulate when multiple forecasts
   horizon are available.
 - [IMPROVED] the `GridStateFromFileWithForecasts` is now able to read forecast from multiple steps
   ahead (provided that it knows the horizons in its constructor)
@@ -137,6 +141,7 @@ Change Log
 - [IMPROVED] it is no more reasonably possible to misuse the `MultifolderWithCache` (for example by
   forgetting to `reset()` the cache): an error will be raised in case the proper function has not been called.
 - [IMPROVED] possibility to pass game rules by instance of object and not by class.
+- [IMPROVED] it should be faster to use the "Simulator" (an useless powerflow was run)
 
 [1.8.1] - 2023-01-11
 ---------------------

grid2op/Environment/BaseEnv.py

@@ -22,6 +22,7 @@
 from grid2op.Action.ActionSpace import ActionSpace
 from grid2op.Observation.baseObservation import BaseObservation
 from grid2op.Observation.observationSpace import ObservationSpace
+from grid2op.Observation.highresSimCounter import HighResSimCounter
 from grid2op.Backend import Backend
 from grid2op.dtypes import dt_int, dt_float, dt_bool
 from grid2op.Space import GridObjects, RandomObject
@@ -257,6 +258,7 @@ def __init__(
         kwargs_observation: Optional[dict] = None,
         observation_bk_class=None,  # type of backend for the observation space
         observation_bk_kwargs=None,  # type of backend for the observation space
+        highres_sim_counter=None,
         _is_test: bool = False,  # TODO not implemented !!
         _init_obs: Optional[BaseObservation] =None
     ):
@@ -509,6 +511,19 @@ def __init__(
         self._observation_bk_class = observation_bk_class
         self._observation_bk_kwargs = observation_bk_kwargs
 
+        if highres_sim_counter is not None:
+            self._highres_sim_counter = highres_sim_counter
+        else:
+            self._highres_sim_counter = HighResSimCounter()
+
+    @property
+    def highres_sim_counter(self):
+        return self._highres_sim_counter
+
+    @property
+    def nb_highres_called(self):
+        return self._highres_sim_counter.nb_highres_called
+
     def _custom_deepcopy_for_copy(self, new_obj, dict_=None):
         if self.__closed:
             raise RuntimeError("Impossible to make a copy of a closed environment !")
@@ -746,6 +761,9 @@ def _custom_deepcopy_for_copy(self, new_obj, dict_=None):
 
         # do not forget !
         new_obj._is_test = self._is_test
+
+        # do not copy it.
+        new_obj._highres_sim_counter = self._highres_sim_counter
 
     def get_path_env(self):
         """

grid2op/Environment/Environment.py

@@ -100,6 +100,7 @@ def __init__(
         kwargs_observation=None,
         observation_bk_class=None,
         observation_bk_kwargs=None,
+        highres_sim_counter=None,
         _init_obs=None,
         _raw_backend_class=None,
         _compat_glop_version=None,
@@ -135,6 +136,7 @@ def __init__(
             kwargs_observation=kwargs_observation,
             observation_bk_class=observation_bk_class,
             observation_bk_kwargs=observation_bk_kwargs,
+            highres_sim_counter=highres_sim_counter,
             _init_obs=_init_obs,
             _is_test=_is_test,  # is this created with "test=True" # TODO not implemented !!
         )

grid2op/Environment/_ObsEnv.py

@@ -70,6 +70,7 @@ def __init__(
         attention_budget_cls=LinearAttentionBudget,
         kwargs_attention_budget={},
         logger=None,
+        highres_sim_counter=None,
         _complete_action_cls=None,
         _ptr_orig_obs_space=None,
     ):
@@ -87,6 +88,7 @@ def __init__(
             kwargs_attention_budget=kwargs_attention_budget,
             kwargs_observation=None,
             logger=logger,
+            highres_sim_counter=highres_sim_counter
         )
         self.__unusable = False  # unsuable if backend cannot be copied
 
@@ -254,11 +256,15 @@ def copy(self):
                            "environment that cannot be copied.")
         backend = self.backend
         self.backend = None
+        _highres_sim_counter = self._highres_sim_counter
+        self._highres_sim_counter = None
         with warnings.catch_warnings():
             warnings.simplefilter("ignore", FutureWarning)
             res = copy.deepcopy(self)
             res.backend = backend.copy()
+        res._highres_sim_counter = _highres_sim_counter
         self.backend = backend
+        self._highres_sim_counter = _highres_sim_counter
         return res
 
     def _reset_to_orig_state(self, obs):
@@ -433,6 +439,7 @@ def simulate(self, action):
                            "environment that cannot be copied.")
         self._ptr_orig_obs_space.simulate_called()
         maybe_exc = self._ptr_orig_obs_space.can_use_simulate()
+        self._highres_sim_counter.add_one()
         if maybe_exc is not None:
             raise maybe_exc
         obs, reward, done, info = self.step(action)

grid2op/Environment/_forecast_env.py

@@ -0,0 +1,22 @@
+# Copyright (c) 2019-2020, RTE (https://www.rte-france.com)
+# See AUTHORS.txt
+# This Source Code Form is subject to the terms of the Mozilla Public License, version 2.0.
+# If a copy of the Mozilla Public License, version 2.0 was not distributed with this file,
+# you can obtain one at http://mozilla.org/MPL/2.0/.
+# SPDX-License-Identifier: MPL-2.0
+# This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems.
+
+from typing import Tuple
+from grid2op.Action import BaseAction
+from grid2op.Observation import BaseObservation
+from grid2op.Environment.Environment import Environment
+
+
+class _ForecastEnv(Environment):
+    """Type of environment that increments the `highres_simulator` when it calls the env.step method.
+    
+    It is used by obs.get_forecast_env.
+    """
+    def step(self, action: BaseAction) -> Tuple[BaseObservation, float, bool, dict]:
+        self._highres_sim_counter += 1
+        return super().step(action)

grid2op/Observation/baseObservation.py

@@ -2341,7 +2341,7 @@ def _aux_add_buses(self, graph, cls, first_id):
         edge_bus_li = [(bus_id,
                         bus_id % cls.n_sub,
                         {"type": "bus_to_substation"})
-                       for id_, bus_id in enumerate(bus_ids) if conn_bus[id_]]
+                       for id_, bus_id in enumerate(bus_ids)]
         graph.add_edges_from(edge_bus_li)
         graph.graph["bus_nodes_id"] = bus_ids
         return bus_ids
@@ -3867,8 +3867,11 @@ def get_simulator(self) -> "grid2op.simulator.Simulator":
             Simulator,
         )  # lazy import to prevent circular references
 
-        res = Simulator(backend=self._obs_env.backend)
+        nb_highres_called = self._obs_env.highres_sim_counter.nb_highres_called
+        res = Simulator(backend=self._obs_env.backend, _highres_sim_counter=self._obs_env._highres_sim_counter)
         res.set_state(self)
+        # it does one simulation when it inits it (calling env.step) so I remove 1 here
+        self._obs_env.highres_sim_counter._HighResSimCounter__nb_highres_called = nb_highres_called
         return res
 
     def _get_array_from_forecast(self, name):
@@ -4178,7 +4181,7 @@ def _make_env_from_arays(self,
                              prod_v: Optional[np.ndarray] = None,
                              maintenance: Optional[np.ndarray] = None):
         from grid2op.Chronics import FromNPY, ChronicsHandler
-        from grid2op.Environment import Environment
+        from grid2op.Environment._forecast_env import _ForecastEnv
         ch = ChronicsHandler(FromNPY,
                              load_p=load_p,
                              load_q=load_q,
@@ -4188,12 +4191,16 @@ def _make_env_from_arays(self,
 
         backend = self._obs_env.backend.copy()
         backend._is_loaded = True
-        res = Environment(**self._ptr_kwargs_env,
-                          backend=backend,
-                          chronics_handler=ch,
-                          parameters=self._obs_env.parameters,
-                          _init_obs=self
-                          )
+        nb_highres_called = self._obs_env.highres_sim_counter.nb_highres_called
+        res = _ForecastEnv(**self._ptr_kwargs_env,
+                           backend=backend,
+                           chronics_handler=ch,
+                           parameters=self._obs_env.parameters,
+                           _init_obs=self,
+                           highres_sim_counter=self._obs_env.highres_sim_counter
+                           )
+        # it does one simulation when it inits it (calling env.step) so I remove 1 here
+        res.highres_sim_counter._HighResSimCounter__nb_highres_called = nb_highres_called
         return res
 
     def change_forecast_parameters(self, params):

grid2op/Observation/highresSimCounter.py

@@ -0,0 +1,27 @@
+# Copyright (c) 2023, RTE (https://www.rte-france.com)
+# See AUTHORS.txt
+# This Source Code Form is subject to the terms of the Mozilla Public License, version 2.0.
+# If a copy of the Mozilla Public License, version 2.0 was not distributed with this file,
+# you can obtain one at http://mozilla.org/MPL/2.0/.
+# SPDX-License-Identifier: MPL-2.0
+# This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems.
+
+
+class HighResSimCounter:
+    """This classes helps to count the total number of call to "high fidelity simulator"
+    the agent made.
+    """
+    def __init__(self) -> None:
+        self.__nb_highres_called = 0
+
+    def __iadd__(self, other):
+        self.__nb_highres_called += int(other)
+        return self
+
+    def add_one(self):
+        self.__nb_highres_called += 1
+
+    @property
+    def nb_highres_called(self):
+        return self.__nb_highres_called
+

grid2op/Observation/observationSpace.py

@@ -133,6 +133,7 @@ def __init__(
         self._update_env_time = 0.0
         self.__nb_simulate_called_this_step = 0
         self.__nb_simulate_called_this_episode = 0
+        self._highres_sim_counter = env.highres_sim_counter
 
         # extra argument to build the observation
         if kwargs_observation is None:
@@ -144,11 +145,10 @@ def __init__(
         self._observation_bk_kwargs = observation_bk_kwargs
 
     def set_real_env_kwargs(self, env):
-        from grid2op.Environment.Environment import Environment
         if not self.with_forecast:
             return 
-
         # I don't need the backend nor the chronics_handler
+        from grid2op.Environment.Environment import Environment
         self._real_env_kwargs = Environment.get_kwargs(env, False, False)
 
         # remove the parameters anyways (the 'forecast parameters will be used
@@ -195,6 +195,7 @@ def _create_obs_env(self, env):
             max_episode_duration=env.max_episode_duration(),
             delta_time_seconds=env.delta_time_seconds,
             logger=self.logger,
+            highres_sim_counter=env.highres_sim_counter,
             _complete_action_cls=env._complete_action_cls,
             _ptr_orig_obs_space=self,
         )
@@ -204,7 +205,8 @@ def _create_obs_env(self, env):
     def _aux_create_backend(self, env, observation_bk_class, observation_bk_kwargs, path_grid_for):
         if observation_bk_kwargs is None:
             observation_bk_kwargs = env.backend._my_kwargs
-        self._backend_obs = observation_bk_class(**observation_bk_kwargs)   
+        observation_bk_class_used = observation_bk_class.init_grid(env.backend)
+        self._backend_obs = observation_bk_class_used(**observation_bk_kwargs)   
         self._backend_obs.set_env_name(env.name)
         self._backend_obs.load_grid(path_grid_for)
         self._backend_obs.assert_grid_correct()
@@ -303,6 +305,10 @@ def nb_simulate_called_this_episode(self):
     def nb_simulate_called_this_step(self):
         return self.__nb_simulate_called_this_step
 
+    @property
+    def total_simulate_simulator_calls(self):
+        return self._highres_sim_counter.total_simulate_simulator_calls
+
     def can_use_simulate(self) -> bool:
         """
         This checks on the rules if the agent has not made too many calls to "obs.simulate" this step
@@ -468,6 +474,11 @@ def _custom_deepcopy_for_copy(self, new_obj):
         new_obj.__nb_simulate_called_this_episode = (
             self.__nb_simulate_called_this_episode
         )
+
+        # never copied (keep track of it)
+        new_obj._highres_sim_counter = (
+            self._highres_sim_counter
+        )
         new_obj._env_param = copy.deepcopy(self._env_param)
 
         # as it's a "pointer" it's init from the env when needed here

grid2op/Runner/aux_fun.py

@@ -45,6 +45,7 @@ def _aux_one_process_parrallel(
     agent_seeds=None,
     max_iter=None,
     add_detailed_output=False,
+    add_nb_highres_sim=False,
 ):
     """this is out of the runner, otherwise it does not work on windows / macos"""
     chronics_handler = ChronicsHandler(
@@ -79,19 +80,14 @@ def _aux_one_process_parrallel(
                 agent_seed=agt_seed,
                 detailed_output=add_detailed_output,
             )
-            (name_chron, cum_reward, nb_time_step, max_ts, episode_data)  = tmp_
+            (name_chron, cum_reward, nb_time_step, max_ts, episode_data, nb_highres_sim)  = tmp_
             id_chron = chronics_handler.get_id()
+            res[i] = (id_chron, name_chron, float(cum_reward), nb_time_step, max_ts)
+
             if add_detailed_output:
-                res[i] = (
-                    id_chron,
-                    name_chron,
-                    float(cum_reward),
-                    nb_time_step,
-                    max_ts,
-                    episode_data,
-                )
-            else:
-                res[i] = (id_chron, name_chron, float(cum_reward), nb_time_step, max_ts)
+                res[i] = (*res[i], episode_data)
+            if add_nb_highres_sim:
+                res[i] = (*res[i], nb_highres_sim)                
         finally:
             env.close()
     return res
@@ -126,6 +122,9 @@ def _aux_run_one_episode(
 
     # reset it
     obs = env.reset()
+    # reset the number of calls to high resolution simulator
+    env._highres_sim_counter._HighResSimCounter__nb_highres_called = 0
+
     # seed and reset the agent
     if agent_seed is not None:
         agent.seed(agent_seed)
@@ -286,8 +285,10 @@ def _aux_run_one_episode(
     episode.to_disk()
     name_chron = env.chronics_handler.get_name()
     return (name_chron, cum_reward,
-            int(time_step), int(max_ts),
-            episode)
+            int(time_step),
+            int(max_ts),
+            episode,
+            env.nb_highres_called)
 
 
 def _aux_make_progress_bar(pbar, total, next_pbar):