Add the possibility to parse historical data to parsers (#1178)

README.md

@@ -314,7 +314,7 @@ It is very simple to add a new country. The Electricity Map backend runs a list
 A parser is a python3 script that is expected to define the method `fetch_production` which returns the production mix at current time, in the format:
 
 ```python
-def fetch_production(country_code='FR', session=None):
+def fetch_production(zone_key='FR', session=None, target_datetime=None, logger=None):
     return {
       'countryCode': 'FR',
       'datetime': '2017-01-01T00:00:00Z',
@@ -339,6 +339,9 @@ def fetch_production(country_code='FR', session=None):
 
 The `session` object is a [python request](http://docs.python-requests.org/en/master/) session that you can re-use to make HTTP requests.
 
+`target_datetime` is used to fetch historical data (when available). `logger` is a `logging.Logger`
+whose output is publicly available so that everyone can monitor correct functioning of the parsers.
+
 The production values should never be negative. Use `None`, or omit the key if a specific production mode is not known.
 Storage values can be both positive (when storing energy) or negative (when the storage is emptied).
 
@@ -374,3 +377,4 @@ from the root directory, replacing `<zone_name>` by the zone identifier of the p
 - `ERROR: for X  Cannot create container for service X: Invalid bind mount spec "<path>": Invalid volume specification: '<volume spec>'`. If you get this error after running `docker-compose up` on Windows, you should tell `docker-compose` to properly understand Windows paths by setting the environment variable `COMPOSE_CONVERT_WINDOWS_PATHS` to `0` by running `setx COMPOSE_CONVERT_WINDOWS_PATHS 0`. You will also need a recent version of `docker-compose`. We have successfully seen this fix work with [v1.13.0-rc4](https://github.com/docker/toolbox/releases/tag/v1.13.0-rc4). More info here: https://github.com/docker/compose/issues/4274.
 
 - No website found at `http://localhost:8000`: This can happen if you're running Docker in a virtual machine. Find out docker's IP using `docker-machine ip default`, and replace `localhost` by your Docker IP when connecting.
+

mockserver/update_state.py

@@ -108,7 +108,7 @@
     obj = json.load(f)['data']
     for dp in production_datapoints:
         production = dict(dp)
-        obj['countries'][dp['countryCode']] = production
+        obj['countries'][dp['zoneKey']] = production
         # Update production
         production['datetime'] = arrow.get(production['datetime']).isoformat()
         # Set random co2 value
@@ -119,12 +119,12 @@
 
     # Update exchanges
     for e in exchanges:
-        exchange_zone_names = e['sortedCountryCodes'].split('->')
+        exchange_zone_names = e['sortedZoneKeys'].split('->')
         e['datetime'] = arrow.get(e['datetime']).isoformat()
-        obj['exchanges'][e['sortedCountryCodes']] = e.copy()
+        obj['exchanges'][e['sortedZoneKeys']] = e.copy()
 
         export_origin_zone_name = exchange_zone_names[0] if e['netFlow'] >= 0 else exchange_zone_names[1]
-        obj['exchanges'][e['sortedCountryCodes']]['co2intensity'] = \
+        obj['exchanges'][e['sortedZoneKeys']]['co2intensity'] = \
             obj['countries'].get(export_origin_zone_name, {}).get('co2intensity')
 
         for z in exchange_zone_names:

parsers/AR.py

@@ -514,22 +514,24 @@ def webparser(req):
     return data_table
 
 
-def fetch_price(country_code='AR', session=None):
+def fetch_price(zone_key='AR', session=None, target_datetime=None, logger=None):
     """
     Requests the last known power price of a given country
     Arguments:
-    country_code (optional) -- used in case a parser is able to fetch multiple countries
+    zone_key (optional) -- used in case a parser is able to fetch multiple countries
     session (optional)      -- request session passed in order to re-use an existing session
     Return:
     A dictionary in the form:
     {
-      'countryCode': 'FR',
+      'zoneKey': 'FR',
       'currency': EUR,
       'datetime': '2017-01-01T00:00:00Z',
       'price': 0.0,
       'source': 'mysource.com'
     }
       """
+    if target_datetime:
+        raise NotImplementedError('This parser is not yet able to parse past dates')
     s = session or requests.Session()
     price_req = s.get(cammesa_url)
     psoup = BeautifulSoup(price_req.content, 'html.parser')
@@ -551,7 +553,7 @@ def fetch_price(country_code='AR', session=None):
     datetime = arrow.now('UTC-3').floor('hour').datetime
 
     data = {
-        'countryCode': country_code,
+        'zoneKey': zone_key,
         'currency': 'ARS',
         'datetime': datetime,
         'price': price,
@@ -591,7 +593,7 @@ def dataformat(junk):
     return formatted
 
 
-def get_thermal(session=None):
+def get_thermal(session, logger):
     """
     Requests thermal generation data then parses and sorts by type.  Nuclear is included.
     Returns a dictionary.
@@ -635,7 +637,7 @@ def get_thermal(session=None):
         try:
             # avoids including titles and headings
             if all((item.isupper(), not item.isalpha(), ' ' not in item)):
-                print('{} is missing from the AR plant mapping!'.format(item))
+                logger.warning('{} is missing from the AR plant mapping!'.format(item))
         except AttributeError:
             # not a string....
             continue
@@ -701,15 +703,17 @@ def get_hydro(session=None):
     return {'hydro': total_hydro_generation}
 
 
-def fetch_production(country_code='AR', session=None):
+def fetch_production(zone_key='AR', session=None, target_datetime=None, logger=None):
     """
     Requests the last known production mix (in MW) of a given country
     Arguments:
-    country_code (optional) -- used in case a parser is able to fetch multiple countries
+    zone_key (optional) -- used in case a parser is able to fetch multiple countries
+    target_datetime: if we want to parser for a specific time and not latest
+    logger: where to log useful information
     Return:
     A dictionary in the form:
     {
-      'countryCode': 'FR',
+      'zoneKey': 'FR',
       'datetime': '2017-01-01T00:00:00Z',
       'production': {
           'biomass': 0.0,
@@ -729,11 +733,14 @@ def fetch_production(country_code='AR', session=None):
       'source': 'mysource.com'
     }
     """
+    if target_datetime is not None:
+        raise NotImplementedError('This parser is not yet able to parse past dates')
+
     gdt = get_datetime(session=None)
-    thermal = get_thermal(session=None)
+    thermal = get_thermal(session, logger)
     hydro = get_hydro(session=None)
     production_mix = {
-        'countryCode': country_code,
+        'zoneKey': zone_key,
         'datetime': gdt['datetime'],
         'production': {
             'biomass': thermal.get('biomass', 0.0),