Merge pull request #24 from BDonnot/l2rpn2019

Fixing import bug

.gitignore

@@ -159,3 +159,9 @@ getting_started/study_agent_getting_started/
 **Untitled.ipynb
 grid2op/tests/test_agent/
 grid2op/tests/start_datetime.info
+
+
+l2rpn_2019/data/
+l2rpn_2019/expe_saved/
+l2rpn_2019/l2rpn2019_utils/data_location.py
+

CHANGELOG.rst

@@ -1,6 +1,13 @@
 Change Log
 =============
 
+[0.5.1] - 2020-01-24
+--------------------
+- [ADDED] extra tag 'all' to install all optional dependencies.
+- [FIXED] issue in the documentation of Observation, voltages are given in kV and not V.
+- [FIXED] a bug in the runner that prevented the right chronics to be read, and output wrong names
+- [FIXED] a bug preventing import if plotting packages where not installed, that causes the documentation to crash.
+
 [0.5.0] - 2020-01-23
 --------------------
 - [BREAKING] Action/Backend has been modified with the implementation of redispatching. If

docs/conf.py

@@ -22,7 +22,7 @@
 author = 'Benjamin Donnot'
 
 # The full version, including alpha/beta/rc tags
-release = '0.5.0'
+release = '0.5.1'
 version = '0.5'
 
 

grid2op/ChronicsHandler.py

@@ -820,7 +820,10 @@ def _assert_correct(self, dict_convert, order_backend):
     def _assert_correct_second_stage(self, pandas_name, dict_convert, key, extra=""):
         for i, el in enumerate(pandas_name):
             if not el in dict_convert[key]:
-                raise ChronicsError("Element named {} is found in the data (column {}) but it is not found on the powergrid for data of type \"{}\".\nData in files  are: {}\nConverter data are: {}".format(el, i+1, key, sorted(list(pandas_name)), sorted(list(dict_convert[key].keys()))))
+                raise ChronicsError("Element named {} is found in the data (column {}) but it is not found on the "
+                                    "powergrid for data of type \"{}\".\nData in files  are: {}\n"
+                                    "Converter data are: {}".format(el, i+1, key, sorted(list(pandas_name)),
+                                                                    sorted(list(dict_convert[key].keys()))))
 
     def _init_date_time(self):
         if os.path.exists(os.path.join(self.path, "start_datetime.info")):
@@ -1018,7 +1021,6 @@ def done(self):
         elif self.max_iter > 0:
             if self.curr_iter > self.max_iter:
                 res = True
-
         return res
 
     def load_next(self):

grid2op/Observation.py

@@ -309,7 +309,7 @@ class Observation(GridObjects):
         The reactive production value of each generator (expressed in MVar).
 
     prod_v: :class:`numpy.ndarray`, dtype:float
-        The voltage magnitude of the bus to which each generator is connected (expressed in V).
+        The voltage magnitude of the bus to which each generator is connected (expressed in kV).
 
     load_p: :class:`numpy.ndarray`, dtype:float
         The active load value of each consumption (expressed in MW).
@@ -318,7 +318,7 @@ class Observation(GridObjects):
         The reactive load value of each consumption (expressed in MVar).
 
     load_v: :class:`numpy.ndarray`, dtype:float
-        The voltage magnitude of the bus to which each consumption is connected (expressed in V).
+        The voltage magnitude of the bus to which each consumption is connected (expressed in kV).
 
     p_or: :class:`numpy.ndarray`, dtype:float
         The active power flow at the origin end of each powerline (expressed in MW).
@@ -327,7 +327,7 @@ class Observation(GridObjects):
         The reactive power flow at the origin end of each powerline (expressed in MVar).
 
     v_or: :class:`numpy.ndarray`, dtype:float
-        The voltage magnitude at the bus to which the origin end of each powerline is connected (expressed in V).
+        The voltage magnitude at the bus to which the origin end of each powerline is connected (expressed in kV).
 
     a_or: :class:`numpy.ndarray`, dtype:float
         The current flow at the origin end of each powerline (expressed in A).
@@ -339,7 +339,7 @@ class Observation(GridObjects):
         The reactive power flow at the extremity end of each powerline (expressed in MVar).
 
     v_ex: :class:`numpy.ndarray`, dtype:float
-        The voltage magnitude at the bus to which the extremity end of each powerline is connected (expressed in V).
+        The voltage magnitude at the bus to which the extremity end of each powerline is connected (expressed in kV).
 
     a_ex: :class:`numpy.ndarray`, dtype:float
         The current flow at the extremity end of each powerline (expressed in A).

grid2op/PlotPlotly.py

@@ -41,10 +41,14 @@
 import cmath
 import pdb
 
+try:
+    from .PlotGraph import BasePlot
+except:
+    from PlotGraph import BasePlot
+
 try:
     import plotly.graph_objects as go
     import seaborn as sns
-    from .PlotGraph import BasePlot
     can_plot = True
 except Exception as e:
     can_plot = False

grid2op/PlotPyGame.py

@@ -14,11 +14,14 @@
 import pdb
 
 try:
-    import pygame
     from .PlotGraph import BasePlot
+except (ModuleNotFoundError, ImportError):
+    from PlotGraph import BasePlot
+
+try:
+    import pygame
     can_plot = True
 except:
-    from PlotGraph import BasePlot
     can_plot = False
     pass
 

grid2op/Runner.py

@@ -507,6 +507,9 @@ def _run_one_episode(env, agent, logger, indx, path_save=None):
         time_act = 0.
         cum_reward = 0.
 
+        # reset the environment
+        obs = env.reset()
+
         if path_save is not None:
             path_save = os.path.abspath(path_save)
             if not os.path.exists(path_save):
@@ -545,6 +548,7 @@ def _run_one_episode(env, agent, logger, indx, path_save=None):
                 dict_params = env.parameters.to_dict()
                 json.dump(obj=dict_params, fp=f, indent=4, sort_keys=True)
 
+        # compute the size and everything if it needs to be stored
         nb_timestep_max = env.chronics_handler.max_timestep()
         efficient_storing = nb_timestep_max > 0
         nb_timestep_max = max(nb_timestep_max, 0)
@@ -563,7 +567,7 @@ def _run_one_episode(env, agent, logger, indx, path_save=None):
 
         beg_ = time.time()
 
-        obs = env.reset()
+
         reward = env.reward_range[0]
         done = False
 

grid2op/Settings_L2RPN2019.py

@@ -9,7 +9,7 @@
 import copy
 import warnings
 
-from datetime import timedelta
+from datetime import timedelta, datetime
 import numpy as np
 import pandas as pd
 
@@ -143,6 +143,10 @@ def initialize(self, order_backend_loads, order_backend_prods, order_backend_lin
         self.hazards = copy.deepcopy(hazards.values[:, np.argsort(order_backend_hazards)])
         self.maintenance = copy.deepcopy(maintenance.values[:, np.argsort(order_backend_maintenance)])
 
+        # date and time
+        datetimes_ = pd.read_csv(os.path.join(self.path, "_N_datetimes{}".format(read_compressed)), sep=self.sep)
+        self.start_datetime = datetime.strptime(datetimes_.iloc[0, 0], "%Y-%b-%d")
+
         # there are maintenance and hazards only if the value in the file is not 0.
         self.maintenance = self.maintenance != 0.
         self.hazards = self.hazards != 0.

grid2op/__init__.py

@@ -6,7 +6,7 @@
 import os
 import pkg_resources
 
-__version__ = '0.5.0'
+__version__ = '0.5.1'
 
 __all__ = ['Action', "BackendPandaPower", "Agent", "Backend", "ChronicsHandler", "Environment", "Exceptions",
            "Observation", "Parameters", "GameRules", "Reward", "Runner", "main", "Utils", "PlotPlotly"]

l2rpn_2019/README.md

@@ -0,0 +1,9 @@
+# Learning to Run, 2019 edition
+
+This folder present all the informations needed to reproduce the L2RPN competition locally. 
+It has helpers to download the data, as well as pre-defined scripts to run locally an agent on
+conditions that are as close as possible to the codalab environment.
+
+It also has the getting started notebooks for this previous competition.
+
+TODO: add getting started of the competition, plus the notebooks to make agent run locally.

l2rpn_2019/l2rpn2019_utils/create_env.py

@@ -0,0 +1,95 @@
+import os
+import sys
+import grid2op
+import grid2op.main
+import argparse
+
+import pdb
+
+from grid2op.ChronicsHandler import GridStateFromFileWithForecasts
+from grid2op.Runner import Runner
+from grid2op.Reward import L2RPNReward
+from grid2op.Settings_L2RPN2019 import L2RPN2019_DICT_NAMES, L2RPN2019_CASEFILE
+from grid2op.Settings_L2RPN2019 import ReadPypowNetData
+
+from datetime import timedelta
+import numpy as np
+import pandas as pd
+import copy
+
+
+PATH_DATA_DEFAULT = os.path.abspath(os.path.join("data", "data_l2rpn_2019"))
+PATH_DATA = PATH_DATA_DEFAULT
+if not os.path.exists(os.path.join("l2rpn2019_utils", "data_location.py")):
+    # the script to download the data has been used, so i use that to retrieve where it has been installed
+    try:
+        from l2rpn2019_utils.data_location import L2RPN_TRAINING_SET as PATH_DATA
+    except Exception as e:
+        # impossible to load the data
+        pass
+
+# todo add confirmation to download data
+
+
+def make_env(path_data=PATH_DATA):
+    env = grid2op.make("l2rpn_2019", chronics_class=path_data)
+    return env
+
+
+def get_submitted_controller(submission_dir):
+    sys.path.append(submission_dir)
+    try:
+        import submission
+    except ImportError:
+        raise ImportError('The submission folder provided (\"{}\") should contain a file submission.py containing your '
+                          'controler named as the class Submission.'.format(submission_dir))
+
+    try:
+        submitted_controler = submission.Submission
+    except:
+        raise Exception('Did not find a class named Submission within submission.py; your submission controler should'
+                        ' be a class named Submission in submission.py file directly within the ZIP submission file.')
+    return submitted_controler
+
+
+def main(path_save=None,
+         submission_dir=".",
+         nb_episode=1,
+         nb_process=1,
+         path_chronics=PATH_DATA,
+         path_parameters=None):
+
+    if path_save is not None:
+        path_save = os.path.abspath(path_save)
+    else:
+        path_save = None
+
+    submitted_controler = get_submitted_controller(submission_dir)
+
+    res = grid2op.main.main(nb_episode=nb_episode,
+                            agent_class=submitted_controler,
+                            path_casefile=L2RPN2019_CASEFILE,
+                            path_chronics=path_chronics,
+                            names_chronics_to_backend=L2RPN2019_DICT_NAMES,
+                            gridStateclass_kwargs={"gridvalueClass": ReadPypowNetData},
+                            reward_class=L2RPNReward,
+                            path_save=path_save,
+                            nb_process=nb_process,
+                            path_parameters=path_parameters)
+    if path_save is not None:
+        print("Done and data saved in : \"{}\"".format(path_save))
+    return res
+
+
+def get_runner(path_chronics=PATH_DATA,
+               submission_dir="."):
+    submitted_controler = get_submitted_controller(submission_dir)
+    runner = Runner(init_grid_path=L2RPN2019_CASEFILE,
+                    path_chron=path_chronics,
+                    names_chronics_to_backend=L2RPN2019_DICT_NAMES,
+                    gridStateclass_kwargs={"gridvalueClass": ReadPypowNetData},
+                    rewardClass=L2RPNReward,
+                    agentClass=submitted_controler)
+    return runner
+
+

l2rpn_2019/l2rpn2019_utils/download_training_data.py

@@ -0,0 +1,84 @@
+"""
+This utility file helps downloading the data for the L2RPN 2019.
+
+Data are stored as a github "release" accessible at the url:
+
+`https://github.com/BDonnot/Grid2Op/releases/download/data_l2rpn_2019/data_l2rpn_2019.tar.bz2`
+
+Once downloaded, the dataset is uncompressed. It is composed of the exact same files provided for the first edition of
+the 2019 L2RPN challenge. There are 1004 chronics of 4 weeks corresponding to january for the fictive grid used
+for this competition.
+
+This script works on MacOs, Linux and windows.
+"""
+import os
+import argparse
+import io
+import sys
+from tqdm import tqdm
+import re
+
+import tarfile
+
+import pdb
+try:
+    import urllib.request
+except Exception as e:
+    raise RuntimeError("Impossible to find library urllib. Please install it.")
+
+URL = "https://github.com/BDonnot/Grid2Op/releases/download/data_l2rpn_2019/data_l2rpn_2019.tar.bz2"
+DEFAULT_PATH_DATA = "data"
+
+
+class DownloadProgressBar(tqdm):
+    """
+    This class is here to show the progress bar when downloading this dataset
+    """
+    def update_to(self, b=1, bsize=1, tsize=None):
+        if tsize is not None:
+            self.total = tsize
+        self.update(b * bsize - self.n)
+
+
+def download_url(url, output_path):
+    """
+    This function download the file located at 'url' and save it to 'output_path'
+    Parameters
+    ----------
+    url: ``str``
+        The url of the file to download
+
+    output_path: ``str``
+        The path where the data will be stored.
+    """
+    with DownloadProgressBar(unit='B', unit_scale=True, miniters=1, desc=url.split('/')[-1]) as t:
+        urllib.request.urlretrieve(url, filename=output_path, reporthook=t.update_to)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description='Launch the evaluation of the Grid2Op ("Grid To Operate") code.')
+    parser.add_argument('--path_save', default=DEFAULT_PATH_DATA, type=str,
+                        help='The path where the data will be downloaded.')
+    args = parser.parse_args()
+    path_data = args.path_save
+
+    if not os.path.exists(path_data):
+        print("Creating path \"{}\" where l2rpn2019 data will be downloaded".format(path_data))
+        os.mkdir(path_data)
+    output_path = os.path.abspath(os.path.join(path_data, "data_l2rpn_2019.tar.bz2"))
+
+    # download the data (with progress bar)
+    print("downloading the training data, this may take a while.")
+    download_url(URL, output_path)
+
+    tar = tarfile.open(output_path, "r:bz2")
+    print("Extract the tar archive in {}".format(path_data))
+    tar.extractall(path_data)
+    tar.close()
+    file_location = os.path.split(os.path.abspath(__file__))[0]
+    output_path_str = re.sub("\\\\", "\\\\\\\\", output_path)
+    with open(os.path.join(file_location, "data_location.py"), "w") as f:
+        f.write("# This file has been automatically generated by 'download_training_data.py' do not modify it, "
+                "nor remove it.\n# If you want to download the training data again, run "
+                "'python l2rpn2019_utils/download_training_data.py'\n\n\n"
+                "L2RPN_TRAINING_SET = '{}'\n".format(output_path_str))